added parents/children to filter model, fixed scraping (+tests)
This commit is contained in:
parent
499bb09125
commit
c701063fb4
3 changed files with 147 additions and 133 deletions
|
|
@ -14,8 +14,10 @@ type Filter struct {
|
|||
Y int `form:"y" yaml:"y" json:"y" validate:"default=0"`
|
||||
Type string `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring"`
|
||||
Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"`
|
||||
Var2 string `form:"var2" yaml:"var2" json:"var2"`
|
||||
Var3 string `form:"var3" yaml:"var3" json:"var3"`
|
||||
Var2 *string `form:"var2" yaml:"var2" json:"var2"`
|
||||
Var3 *string `form:"var3" yaml:"var3" json:"var3"`
|
||||
Parents []*Filter `gorm:"-:all"`
|
||||
Children []*Filter `gorm:"-:all"`
|
||||
Results []string `gorm:"-:all"`
|
||||
}
|
||||
|
||||
|
|
|
|||
62
scraping.go
62
scraping.go
|
|
@ -1,12 +1,17 @@
|
|||
package main
|
||||
|
||||
/*
|
||||
func getFilterResults(filter *Filter) {
|
||||
getFilterResult(filter)
|
||||
for _, filter := range filter.Filters {
|
||||
getFilterResults(&filter)
|
||||
}
|
||||
}
|
||||
import (
|
||||
"bytes"
|
||||
"log"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/andybalholm/cascadia"
|
||||
"github.com/antchfx/htmlquery"
|
||||
"github.com/tidwall/gjson"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func getFilterResult(filter *Filter) {
|
||||
switch {
|
||||
|
|
@ -40,11 +45,12 @@ func getFilterResult(filter *Filter) {
|
|||
}
|
||||
|
||||
func getFilterResultXPath(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parents for", filter.Type)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
doc, err := htmlquery.Parse(strings.NewReader(result))
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
|
|
@ -58,25 +64,29 @@ func getFilterResultXPath(filter *Filter) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultJSON(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
for _, match := range gjson.Get(result, filter.Var1).Array() {
|
||||
filter.Results = append(filter.Results, match.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultCSS(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
doc, err := html.Parse(strings.NewReader(result))
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
|
|
@ -95,13 +105,15 @@ func getFilterResultCSS(filter *Filter) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultReplace(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
r, err := regexp.Compile(filter.Var1)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
|
|
@ -114,9 +126,10 @@ func getFilterResultReplace(filter *Filter) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultMatch(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
return
|
||||
}
|
||||
|
|
@ -125,7 +138,8 @@ func getFilterResultMatch(filter *Filter) {
|
|||
log.Print(err)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
log.Println(">", result)
|
||||
for _, str := range r.FindAllString(result, -1) {
|
||||
log.Println(">>", str)
|
||||
|
|
@ -133,13 +147,15 @@ func getFilterResultMatch(filter *Filter) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultSubstring(filter *Filter) {
|
||||
if filter.Parent == nil {
|
||||
if filter.Parents == nil {
|
||||
log.Println("Filter", filter.Name, "called without parent for", filter.Type)
|
||||
return
|
||||
}
|
||||
for _, result := range filter.Parent.Results {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
substrings := strings.Split(filter.Var1, ",")
|
||||
var sb strings.Builder
|
||||
asRunes := []rune(result)
|
||||
|
|
@ -148,7 +164,6 @@ func getFilterResultSubstring(filter *Filter) {
|
|||
if strings.Contains(substring, ":") {
|
||||
from_to := strings.Split(substring, ":")
|
||||
if len(from_to) != 2 {
|
||||
filter.Results = filter.Parent.Results
|
||||
return
|
||||
}
|
||||
fromStr := from_to[0]
|
||||
|
|
@ -159,7 +174,6 @@ func getFilterResultSubstring(filter *Filter) {
|
|||
from64, err := strconv.ParseInt(fromStr, 10, 32)
|
||||
var from = int(from64)
|
||||
if hasFrom && err != nil {
|
||||
filter.Results = filter.Parent.Results
|
||||
return
|
||||
} else if from < 0 {
|
||||
from = len(asRunes) + from
|
||||
|
|
@ -172,7 +186,6 @@ func getFilterResultSubstring(filter *Filter) {
|
|||
to64, err := strconv.ParseInt(toStr, 10, 32)
|
||||
var to = int(to64)
|
||||
if hasTo && err != nil {
|
||||
filter.Results = filter.Parent.Results
|
||||
return
|
||||
} else if to < 0 {
|
||||
to = len(asRunes) + to
|
||||
|
|
@ -187,7 +200,6 @@ func getFilterResultSubstring(filter *Filter) {
|
|||
} else {
|
||||
pos, err := strconv.ParseInt(substring, 10, 32)
|
||||
if err != nil || pos < 0 {
|
||||
filter.Results = filter.Parent.Results
|
||||
return
|
||||
}
|
||||
sb.WriteRune(asRunes[pos])
|
||||
|
|
@ -196,4 +208,4 @@ func getFilterResultSubstring(filter *Filter) {
|
|||
filter.Results = append(filter.Results, sb.String())
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ func TestFilterXPath(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s", test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{HTML_STRING},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{HTML_STRING}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
@ -86,8 +86,8 @@ func TestFilterJSON(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s", test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{JSON_STRING},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{JSON_STRING}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
@ -117,8 +117,8 @@ func TestFilterCSS(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s", test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{HTML_STRING},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{HTML_STRING}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
@ -154,8 +154,8 @@ func TestFilterReplace(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s %s", test.Input, test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{test.Input},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{test.Input}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
@ -186,8 +186,8 @@ func TestFilterMatch(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s", test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{test.Input},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{test.Input}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
@ -237,8 +237,8 @@ func TestFilterSubstring(t *testing.T) {
|
|||
testname := fmt.Sprintf("%s %s", test.Input, test.Query)
|
||||
t.Run(testname, func(t *testing.T) {
|
||||
filter := Filter{
|
||||
Parent: &Filter{
|
||||
Results: []string{test.Input},
|
||||
Parents: []*Filter{
|
||||
{Results: []string{test.Input}},
|
||||
},
|
||||
Var1: test.Query,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue