broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

filter.Var2 is now a value instead of a pointer

Browse source
This commit is contained in:
BroodjeAap 2023-03-27 19:08:51 +00:00
parent 0c52878b08
commit 7d377e905a
6 changed files with 36 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
models/filter.go
View file

@ -17,13 +17,12 @@ type Filter struct {
Y int `form:"y" yaml:"y" json:"y" validate:"default=0"`
Type FilterType `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math store condition cron"`
Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"`
Var2 *string `form:"var2" yaml:"var2" json:"var2"`
Var2 string `form:"var2" yaml:"var2" json:"var2"`
Parents []*Filter `gorm:"-:all"`
Children []*Filter `gorm:"-:all"`
Results []string `gorm:"-:all"`
Logs []string `gorm:"-:all"`
CronEntry *cron.Entry `gorm:"-:all"`
Enabled string `gorm:"-:all"` // lazy fix for not being able to pointer deref in golang template....
}
func (filter *Filter) Log(v ...any) {

1
todo.md
View file

@ -2,4 +2,3 @@
- add time 'jitter' to schedule filter
- var2 with another duration string, var1 + (var2 * random) duration
- remove startup delay?
- change Filter.Var2 from pointer to just value?

60
web/scraping.go
View file

@ -97,7 +97,7 @@ func ProcessFilters(filters []Filter, web *Web, watch *Watch, debug bool, schedu
// for schedule filters during editing, just check the schedule string and set to processed
if debug && filter.Type == "cron" {
if filter.Var2 != nil && *filter.Var2 == "no" {
if filter.Var2 == "no" {
filter.Log("Schedule is disabled")
}
processedMap[filter.ID] = true
@ -380,11 +380,7 @@ func getURLContent(filter *Filter, fetchURL string) (string, error) {
// getFilterResultBrowserlessURL Fetches the given URL and outputs the HTTP response through Browserless
func getFilterResultBrowserlessURL(filter *Filter, urlCache map[string]string, debug bool) {
if filter.Var2 == nil {
filter.Log("filter.Var2 == nil")
return
}
fetchURL := *filter.Var2
fetchURL := filter.Var2
val, exists := urlCache["b"+fetchURL]
if debug && exists {
filter.Results = append(filter.Results, val)
@ -499,10 +495,7 @@ func getBrowserlessFunctionContent(filter *Filter, result string) (string, error
return "", errors.New("browserless.url not set")
}
browserlessURL := viper.GetString("browserless.url")
if filter.Var2 == nil {
return "", errors.New("filter.Var2 == nil")
}
code := *filter.Var2
code := filter.Var2
data := struct {
Code string `json:"code"`
Context BrowserlessContext `json:"context"`
@ -533,10 +526,7 @@ func getBrowserlessFunctionContent(filter *Filter, result string) (string, error
// getFilterResultXPath Filters the results of its parents based on an XPath query
func getFilterResultXPath(filter *Filter) {
selectType := "node"
if filter.Var2 != nil {
selectType = *filter.Var2
}
selectType := filter.Var2
for _, parent := range filter.Parents {
for _, result := range parent.Results {
doc, err := htmlquery.Parse(strings.NewReader(result))
@ -596,10 +586,7 @@ func getFilterResultJSON(filter *Filter) {
// getFilterResultCSS Filters the results of its parents based on an CSS query
func getFilterResultCSS(filter *Filter) {
selectType := "node"
if filter.Var2 != nil {
selectType = *filter.Var2
}
selectType := filter.Var2
for _, parent := range filter.Parents {
for _, result := range parent.Results {
doc, err := html.Parse(strings.NewReader(result))
@ -659,11 +646,7 @@ func getFilterResultReplace(filter *Filter) {
}
for _, parent := range filter.Parents {
for _, result := range parent.Results {
if filter.Var2 == nil {
filter.Results = append(filter.Results, r.ReplaceAllString(result, ""))
} else {
filter.Results = append(filter.Results, r.ReplaceAllString(result, *filter.Var2))
}
filter.Results = append(filter.Results, r.ReplaceAllString(result, filter.Var2))
}
}
}
@ -759,7 +742,7 @@ func getFilterResultSubstring(filter *Filter) {
// getFilterResultContains performs a regex contains on all the results of its parents
func getFilterResultContains(filter *Filter) {
r, err := regexp.Compile(filter.Var1)
invert, err := strconv.ParseBool(*filter.Var2)
invert, err := strconv.ParseBool(filter.Var2)
if err != nil {
invert = false
}
@ -893,14 +876,10 @@ func roundFloat(val float64, precision uint) float64 {
// getFilterResultRound outputs the rounded value of every numerical input
func getFilterResultRound(filter *Filter) {
var decimals int64 = 0
if filter.Var2 != nil {
d, err := strconv.ParseInt(*filter.Var2, 10, 32)
if err != nil {
decimals = 0
} else {
d, err := strconv.ParseInt(filter.Var2, 10, 32)
if err == nil {
decimals = d
}
}
for _, parent := range filter.Parents {
for _, result := range parent.Results {
@ -1023,13 +1002,9 @@ func getFilterResultConditionLowest(filter *Filter, db *gorm.DB) {
// getFilterResultConditionLowerThan outputs any input that is lower than a given value
func getFilterResultConditionLowerThan(filter *Filter) {
if filter.Var2 == nil {
filter.Log("No threshold given")
return
}
threshold, err := strconv.ParseFloat(*filter.Var2, 64)
threshold, err := strconv.ParseFloat(filter.Var2, 64)
if err != nil {
filter.Log("Could not convert convert threshold to number: '", *filter.Var2, "'")
filter.Log("Could not convert convert threshold to number: '", filter.Var2, "'")
return
}
for _, parent := range filter.Parents {
@ -1117,13 +1092,9 @@ func getFilterResultConditionHighest(filter *Filter, db *gorm.DB) {
// getFilterResultConditionHigherThan outputs any input that is higher than a given value
func getFilterResultConditionHigherThan(filter *Filter) {
if filter.Var2 == nil {
filter.Log("No threshold given for Higher Than Filter")
return
}
threshold, err := strconv.ParseFloat(*filter.Var2, 64)
threshold, err := strconv.ParseFloat(filter.Var2, 64)
if err != nil {
filter.Log("Could not convert convert threshold to number: '", *filter.Var2, "'")
filter.Log("Could not convert convert threshold to number: '", filter.Var2, "'")
return
}
for _, parent := range filter.Parents {
@ -1195,10 +1166,7 @@ func notifyFilter(filters []Filter, filter *Filter, watch *Watch, web *Web, debu
filter.Results = append(filter.Results, buffer.String())
} else {
notifier := filter.Var2
if notifier == nil {
return
}
web.notify(*notifier, buffer.String())
web.notify(notifier, buffer.String())
}
}

38
web/scraping_test.go
View file

@ -55,7 +55,6 @@ func DeepEqualStringSlice(a []string, b []string) bool {
}
func TestFilterXPathNode(t *testing.T) {
var2 := "node"
var tests = []struct {
Query string
Want []string
@ -79,7 +78,7 @@ func TestFilterXPathNode(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "node",
}
getFilterResultXPath(
&filter,
@ -92,7 +91,6 @@ func TestFilterXPathNode(t *testing.T) {
}
func TestFilterXPathInnerHTML(t *testing.T) {
var2 := "inner"
var tests = []struct {
Query string
Want []string
@ -115,7 +113,7 @@ func TestFilterXPathInnerHTML(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "inner",
}
getFilterResultXPath(
&filter,
@ -128,7 +126,6 @@ func TestFilterXPathInnerHTML(t *testing.T) {
}
func TestFilterXPathAttributes(t *testing.T) {
var2 := "attr"
var tests = []struct {
Query string
Want []string
@ -150,7 +147,7 @@ func TestFilterXPathAttributes(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "attr",
}
getFilterResultXPath(
&filter,
@ -194,7 +191,6 @@ func TestFilterJSON(t *testing.T) {
}
func TestFilterCSSNode(t *testing.T) {
var2 := "node"
var tests = []struct {
Query string
Want []string
@ -216,7 +212,7 @@ func TestFilterCSSNode(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "node",
}
getFilterResultCSS(
&filter,
@ -229,7 +225,6 @@ func TestFilterCSSNode(t *testing.T) {
}
func TestFilterCSSInnerHTML(t *testing.T) {
var2 := "inner"
var tests = []struct {
Query string
Want []string
@ -251,7 +246,7 @@ func TestFilterCSSInnerHTML(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "inner",
}
getFilterResultCSS(
&filter,
@ -264,7 +259,6 @@ func TestFilterCSSInnerHTML(t *testing.T) {
}
func TestFilterCSSAttributes(t *testing.T) {
var2 := "attr"
var tests = []struct {
Query string
Want []string
@ -286,7 +280,7 @@ func TestFilterCSSAttributes(t *testing.T) {
{Results: []string{HTML_STRING}},
},
Var1: test.Query,
Var2: &var2,
Var2: "attr",
}
getFilterResultCSS(
&filter,
@ -342,7 +336,7 @@ func TestFilterReplace(t *testing.T) {
{Results: []string{test.Input}},
},
Var1: test.Var1,
Var2: &test.Var2,
Var2: test.Var2,
}
getFilterResultReplace(
&filter,
@ -486,7 +480,7 @@ func TestFilterContains(t *testing.T) {
{Results: test.Input},
},
Var1: test.Query,
Var2: &test.Invert,
Var2: test.Invert,
}
getFilterResultContains(
&filter,
@ -685,7 +679,7 @@ func TestFilterRound(t *testing.T) {
Parents: []*Filter{
{Results: test.Input},
},
Var2: &test.Query,
Var2: test.Query,
}
getFilterResultRound(
&filter,
@ -791,7 +785,7 @@ func TestConditionDiff(t *testing.T) {
filter := Filter{
WatchID: test.WatchID,
Name: "Test",
Var2: &testName,
Var2: testName,
Parents: []*Filter{
{Results: test.Input},
},
@ -911,7 +905,7 @@ func TestConditionLowerLast(t *testing.T) {
filter := Filter{
WatchID: test.WatchID,
Name: testName,
Var2: &testName,
Var2: testName,
Parents: []*Filter{
{Results: test.Input},
},
@ -1018,7 +1012,7 @@ func TestConditionLowest(t *testing.T) {
filter := Filter{
WatchID: test.WatchID,
Name: testName,
Var2: &testName,
Var2: testName,
Parents: []*Filter{
{Results: test.Input},
},
@ -1060,7 +1054,7 @@ func TestFilterLowerThan(t *testing.T) {
Results: test.Input,
},
},
Var2: &test.Threshold,
Var2: test.Threshold,
}
getFilterResultConditionLowerThan(
&filter,
@ -1173,7 +1167,7 @@ func TestConditionHigherLast(t *testing.T) {
filter := Filter{
WatchID: test.WatchID,
Name: testName,
Var2: &testName,
Var2: testName,
Parents: []*Filter{
{Results: test.Input},
},
@ -1280,7 +1274,7 @@ func TestConditionHighest(t *testing.T) {
filter := Filter{
WatchID: test.WatchID,
Name: testName,
Var2: &testName,
Var2: testName,
Parents: []*Filter{
{Results: test.Input},
},
@ -1322,7 +1316,7 @@ func TestFilterHigherThan(t *testing.T) {
Results: test.Input,
},
},
Var2: &test.Threshold,
Var2: test.Threshold,
}
getFilterResultConditionHigherThan(
&filter,

2
web/templates/schedules.html
View file

@ -28,7 +28,7 @@ GoWatch
{{ else }}
<td colspan="2" class="h3">Not scheduled</td>
{{ end }}
{{ if eq $scheduleFilter.Enabled "yes" }}
{{ if eq $scheduleFilter.Var2 "yes" }}
<td class="h2">
<input class="form-check-input" id="schedules_{{ $scheduleFilter.ID }}" type="checkbox" value="{{ $scheduleFilter.ID }}" name="schedules" checked>
</td>

6
web/web.go
View file

@ -88,7 +88,7 @@ func (web *Web) addCronJobIfCronFilter(filter *Filter, startup bool) {
if filter.Type != "cron" {
return
}
if filter.Var2 != nil && *filter.Var2 == "no" {
if filter.Var2 == "no" {
return
}
scheduleSplit := strings.Split(filter.Var1, "+")
@ -538,7 +538,6 @@ func (web *Web) schedulesView(c *gin.Context) {
entry := web.cron.Entry(entryID)
filter.CronEntry = &entry
}
filter.Enabled = *filter.Var2
watch := watchMap[filter.WatchID]
watchSchedules[watch] = append(watchSchedules[watch], filter)
}
@ -569,8 +568,7 @@ func (web *Web) schedulesUpdate(c *gin.Context) {
web.cron.Remove(entryID)
delete(web.cronWatch, cronFilter.ID)
} else if !exist && checked {
yes := "yes"
cronFilter.Var2 = &yes
cronFilter.Var2 = "yes"
web.addCronJobIfCronFilter(cronFilter, false)
}
}