broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added parents/children to filter model, fixed scraping (+tests)

Browse source
This commit is contained in:
BroodjeAap 2022-09-25 15:52:53 +00:00
parent 499bb09125
commit c701063fb4
3 changed files with 147 additions and 133 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
models.go
View file

@ -14,8 +14,10 @@ type Filter struct {
Y int `form:"y" yaml:"y" json:"y" validate:"default=0"` Y int `form:"y" yaml:"y" json:"y" validate:"default=0"`
Type string `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring"` Type string `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring"`
Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"` Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"`
Var2 string `form:"var2" yaml:"var2" json:"var2"` Var2 *string `form:"var2" yaml:"var2" json:"var2"`
Var3 string `form:"var3" yaml:"var3" json:"var3"` Var3 *string `form:"var3" yaml:"var3" json:"var3"`
Parents []*Filter `gorm:"-:all"`
Children []*Filter `gorm:"-:all"`
Results []string `gorm:"-:all"` Results []string `gorm:"-:all"`
} }

62
scraping.go
View file

@ -1,12 +1,17 @@
package main package main
/* import (
func getFilterResults(filter *Filter) { "bytes"
getFilterResult(filter) "log"
for _, filter := range filter.Filters { "regexp"
getFilterResults(&filter) "strconv"
} "strings"
}
"github.com/andybalholm/cascadia"
"github.com/antchfx/htmlquery"
"github.com/tidwall/gjson"
"golang.org/x/net/html"
)
func getFilterResult(filter *Filter) { func getFilterResult(filter *Filter) {
switch { switch {
@ -40,11 +45,12 @@ func getFilterResult(filter *Filter) {
} }
func getFilterResultXPath(filter *Filter) { func getFilterResultXPath(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parents for", filter.Type)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
doc, err := htmlquery.Parse(strings.NewReader(result)) doc, err := htmlquery.Parse(strings.NewReader(result))
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -58,25 +64,29 @@ func getFilterResultXPath(filter *Filter) {
} }
} }
} }
}
func getFilterResultJSON(filter *Filter) { func getFilterResultJSON(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parent for", filter.Type)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
for _, match := range gjson.Get(result, filter.Var1).Array() { for _, match := range gjson.Get(result, filter.Var1).Array() {
filter.Results = append(filter.Results, match.String()) filter.Results = append(filter.Results, match.String())
} }
} }
} }
}
func getFilterResultCSS(filter *Filter) { func getFilterResultCSS(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parent for", filter.Type)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
doc, err := html.Parse(strings.NewReader(result)) doc, err := html.Parse(strings.NewReader(result))
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -95,13 +105,15 @@ func getFilterResultCSS(filter *Filter) {
} }
} }
} }
}
func getFilterResultReplace(filter *Filter) { func getFilterResultReplace(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parent for", filter.Type)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
r, err := regexp.Compile(filter.Var1) r, err := regexp.Compile(filter.Var1)
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -114,9 +126,10 @@ func getFilterResultReplace(filter *Filter) {
} }
} }
} }
}
func getFilterResultMatch(filter *Filter) { func getFilterResultMatch(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parent for", filter.Type)
return return
} }
@ -125,7 +138,8 @@ func getFilterResultMatch(filter *Filter) {
log.Print(err) log.Print(err)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
log.Println(">", result) log.Println(">", result)
for _, str := range r.FindAllString(result, -1) { for _, str := range r.FindAllString(result, -1) {
log.Println(">>", str) log.Println(">>", str)
@ -133,13 +147,15 @@ func getFilterResultMatch(filter *Filter) {
} }
} }
} }
}
func getFilterResultSubstring(filter *Filter) { func getFilterResultSubstring(filter *Filter) {
if filter.Parent == nil { if filter.Parents == nil {
log.Println("Filter", filter.Name, "called without parent for", filter.Type) log.Println("Filter", filter.Name, "called without parent for", filter.Type)
return return
} }
for _, result := range filter.Parent.Results { for _, parent := range filter.Parents {
for _, result := range parent.Results {
substrings := strings.Split(filter.Var1, ",") substrings := strings.Split(filter.Var1, ",")
var sb strings.Builder var sb strings.Builder
asRunes := []rune(result) asRunes := []rune(result)
@ -148,7 +164,6 @@ func getFilterResultSubstring(filter *Filter) {
if strings.Contains(substring, ":") { if strings.Contains(substring, ":") {
from_to := strings.Split(substring, ":") from_to := strings.Split(substring, ":")
if len(from_to) != 2 { if len(from_to) != 2 {
filter.Results = filter.Parent.Results
return return
} }
fromStr := from_to[0] fromStr := from_to[0]
@ -159,7 +174,6 @@ func getFilterResultSubstring(filter *Filter) {
from64, err := strconv.ParseInt(fromStr, 10, 32) from64, err := strconv.ParseInt(fromStr, 10, 32)
var from = int(from64) var from = int(from64)
if hasFrom && err != nil { if hasFrom && err != nil {
filter.Results = filter.Parent.Results
return return
} else if from < 0 { } else if from < 0 {
from = len(asRunes) + from from = len(asRunes) + from
@ -172,7 +186,6 @@ func getFilterResultSubstring(filter *Filter) {
to64, err := strconv.ParseInt(toStr, 10, 32) to64, err := strconv.ParseInt(toStr, 10, 32)
var to = int(to64) var to = int(to64)
if hasTo && err != nil { if hasTo && err != nil {
filter.Results = filter.Parent.Results
return return
} else if to < 0 { } else if to < 0 {
to = len(asRunes) + to to = len(asRunes) + to
@ -187,7 +200,6 @@ func getFilterResultSubstring(filter *Filter) {
} else { } else {
pos, err := strconv.ParseInt(substring, 10, 32) pos, err := strconv.ParseInt(substring, 10, 32)
if err != nil || pos < 0 { if err != nil || pos < 0 {
filter.Results = filter.Parent.Results
return return
} }
sb.WriteRune(asRunes[pos]) sb.WriteRune(asRunes[pos])
@ -196,4 +208,4 @@ func getFilterResultSubstring(filter *Filter) {
filter.Results = append(filter.Results, sb.String()) filter.Results = append(filter.Results, sb.String())
} }
} }
*/ }

24
scraping_test.go
View file

@ -56,8 +56,8 @@ func TestFilterXPath(t *testing.T) {
testname := fmt.Sprintf("%s", test.Query) testname := fmt.Sprintf("%s", test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{HTML_STRING}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
} }
@ -86,8 +86,8 @@ func TestFilterJSON(t *testing.T) {
testname := fmt.Sprintf("%s", test.Query) testname := fmt.Sprintf("%s", test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{JSON_STRING}, {Results: []string{JSON_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
} }
@ -117,8 +117,8 @@ func TestFilterCSS(t *testing.T) {
testname := fmt.Sprintf("%s", test.Query) testname := fmt.Sprintf("%s", test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{HTML_STRING}, {Results: []string{HTML_STRING}},
}, },
Var1: test.Query, Var1: test.Query,
} }
@ -154,8 +154,8 @@ func TestFilterReplace(t *testing.T) {
testname := fmt.Sprintf("%s %s", test.Input, test.Query) testname := fmt.Sprintf("%s %s", test.Input, test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{test.Input}, {Results: []string{test.Input}},
}, },
Var1: test.Query, Var1: test.Query,
} }
@ -186,8 +186,8 @@ func TestFilterMatch(t *testing.T) {
testname := fmt.Sprintf("%s", test.Query) testname := fmt.Sprintf("%s", test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{test.Input}, {Results: []string{test.Input}},
}, },
Var1: test.Query, Var1: test.Query,
} }
@ -237,8 +237,8 @@ func TestFilterSubstring(t *testing.T) {
testname := fmt.Sprintf("%s %s", test.Input, test.Query) testname := fmt.Sprintf("%s %s", test.Input, test.Query)
t.Run(testname, func(t *testing.T) { t.Run(testname, func(t *testing.T) {
filter := Filter{ filter := Filter{
Parent: &Filter{ Parents: []*Filter{
Results: []string{test.Input}, {Results: []string{test.Input}},
}, },
Var1: test.Query, Var1: test.Query,
} }