diff --git a/go.mod b/go.mod index afcd8bb..22cce88 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-playground/validator/v10 v10.11.0 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 github.com/spf13/viper v1.12.0 + github.com/tidwall/gjson v1.14.2 golang.org/x/net v0.0.0-20220722155237-a158d28d115b gorm.io/driver/sqlite v1.3.6 gorm.io/gorm v1.23.8 @@ -41,6 +42,8 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.3.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/go.sum b/go.sum index 648563c..3cec9f9 100644 --- a/go.sum +++ b/go.sum @@ -228,6 +228,12 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= diff --git a/models.go b/models.go index 8416b65..7dd06e3 100644 --- a/models.go +++ b/models.go @@ -34,7 +34,7 @@ type Filter struct { FilterGroupID uint `form:"filter_group_id" yaml:"filter_group_id" binding:"required"` FilterGroup *FilterGroup Name string `form:"filter_name" yaml:"filter_name" binding:"required" validate:"min=1"` - Type string `form:"filter_type" yaml:"filter_type" binding:"required" validate:"oneof=xpath css replace match substring"` + Type string `form:"filter_type" yaml:"filter_type" binding:"required" validate:"oneof=xpath json css replace match substring"` From string `form:"from" yaml:"from" binding:"required"` To string `form:"to" yaml:"to" binding:"required"` } diff --git a/scraping.go b/scraping.go index 64af8f7..3e10e36 100644 --- a/scraping.go +++ b/scraping.go @@ -11,6 +11,7 @@ import ( "github.com/andybalholm/cascadia" "github.com/antchfx/htmlquery" + "github.com/tidwall/gjson" "golang.org/x/net/html" ) @@ -44,6 +45,10 @@ func getFilterResult(s string, filter *Filter, newStrings *[]string) { { getFilterResultXPath(s, filter, newStrings) } + case filter.Type == "json": + { + getFilterResultJSON(s, filter, newStrings) + } case filter.Type == "css": { getFilterResultCSS(s, filter, newStrings) @@ -79,6 +84,12 @@ func getFilterResultXPath(s string, filter *Filter, newStrings *[]string) { } } +func getFilterResultJSON(s string, filter *Filter, newStrings *[]string) { + for _, result := range gjson.GetMany(filter.From) { + *newStrings = append(*newStrings, result.String()) + } +} + func getFilterResultCSS(s string, filter *Filter, newStrings *[]string) { doc, err := html.Parse(strings.NewReader(s)) if err != nil { diff --git a/templates/editGroup.html b/templates/editGroup.html index a1b3687..1fa8ea6 100644 --- a/templates/editGroup.html +++ b/templates/editGroup.html @@ -109,7 +109,7 @@ -->