added json filter
This commit is contained in:
parent
9e578a9e19
commit
8edf5eb642
5 changed files with 68 additions and 3 deletions
3
go.mod
3
go.mod
|
|
@ -10,6 +10,7 @@ require (
|
|||
github.com/go-playground/validator/v10 v10.11.0
|
||||
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
|
||||
github.com/spf13/viper v1.12.0
|
||||
github.com/tidwall/gjson v1.14.2
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b
|
||||
gorm.io/driver/sqlite v1.3.6
|
||||
gorm.io/gorm v1.23.8
|
||||
|
|
@ -41,6 +42,8 @@ require (
|
|||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.3.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/ugorji/go/codec v1.2.7 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -228,6 +228,12 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
|
|||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI=
|
||||
github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
|
||||
github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
|
||||
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
|
||||
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ type Filter struct {
|
|||
FilterGroupID uint `form:"filter_group_id" yaml:"filter_group_id" binding:"required"`
|
||||
FilterGroup *FilterGroup
|
||||
Name string `form:"filter_name" yaml:"filter_name" binding:"required" validate:"min=1"`
|
||||
Type string `form:"filter_type" yaml:"filter_type" binding:"required" validate:"oneof=xpath css replace match substring"`
|
||||
Type string `form:"filter_type" yaml:"filter_type" binding:"required" validate:"oneof=xpath json css replace match substring"`
|
||||
From string `form:"from" yaml:"from" binding:"required"`
|
||||
To string `form:"to" yaml:"to" binding:"required"`
|
||||
}
|
||||
|
|
|
|||
11
scraping.go
11
scraping.go
|
|
@ -11,6 +11,7 @@ import (
|
|||
|
||||
"github.com/andybalholm/cascadia"
|
||||
"github.com/antchfx/htmlquery"
|
||||
"github.com/tidwall/gjson"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
|
|
@ -44,6 +45,10 @@ func getFilterResult(s string, filter *Filter, newStrings *[]string) {
|
|||
{
|
||||
getFilterResultXPath(s, filter, newStrings)
|
||||
}
|
||||
case filter.Type == "json":
|
||||
{
|
||||
getFilterResultJSON(s, filter, newStrings)
|
||||
}
|
||||
case filter.Type == "css":
|
||||
{
|
||||
getFilterResultCSS(s, filter, newStrings)
|
||||
|
|
@ -79,6 +84,12 @@ func getFilterResultXPath(s string, filter *Filter, newStrings *[]string) {
|
|||
}
|
||||
}
|
||||
|
||||
func getFilterResultJSON(s string, filter *Filter, newStrings *[]string) {
|
||||
for _, result := range gjson.GetMany(filter.From) {
|
||||
*newStrings = append(*newStrings, result.String())
|
||||
}
|
||||
}
|
||||
|
||||
func getFilterResultCSS(s string, filter *Filter, newStrings *[]string) {
|
||||
doc, err := html.Parse(strings.NewReader(s))
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@
|
|||
-->
|
||||
|
||||
<div class="modal fade" id="newFilterModal" tabindex="-1" aria-labelledby="newFilterModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="newFilterModalLabel">Add Filter</h5>
|
||||
|
|
@ -120,6 +120,9 @@
|
|||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link active" id="xpath-tab" data-bs-toggle="tab" data-bs-target="#xpath-tab-pane" aria-controls="xpath-tab-pane" type="button" role="tab" aria-selected="true">XPath</button>
|
||||
</li>
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link" id="json-tab" data-bs-toggle="tab" data-bs-target="#json-tab-pane" aria-controls="json-tab-pane" type="button" role="tab" aria-selected="false">JSON</button>
|
||||
</li>
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link" id="css-tab" data-bs-toggle="tab" data-bs-target="#css-tab-pane" aria-controls="css-tab-pane" type="button" role="tab" aria-selected="false">CSS</button>
|
||||
</li>
|
||||
|
|
@ -153,6 +156,25 @@
|
|||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="tab-pane fade show" id="json-tab-pane" role="tabpanel" aria-labelledby="json-tab" tabindex="0">
|
||||
<form action="/filter/create" method="post">
|
||||
<div class="mb-3 m-3 row">
|
||||
<input type="hidden" name="filter_group_id" value="{{ .Group.ID }}" >
|
||||
<input type="hidden" name="filter_type" value="json" >
|
||||
<input type="hidden" name="to" value="-" >
|
||||
|
||||
<label for="nameInput" class="col-sm-2 col-form-label">Name:</label>
|
||||
<div class="col-sm-10 p-2">
|
||||
<input type="text" class="form-control" name="filter_name" id="nameInput" placeholder="Name">
|
||||
</div>
|
||||
<label for="fromInput" class="col-sm-2 col-form-label">JSON:</label>
|
||||
<div class="col-sm-10 p-2">
|
||||
<input type="text" class="form-control" name="from" id="fromInput" placeholder="products.#.price">
|
||||
</div>
|
||||
<button class="btn btn-primary mt-4">Add Filter</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="tab-pane fade" id="css-tab-pane" role="tabpanel" aria-labelledby="css-tab" tabindex="0">
|
||||
<form action="/filter/create" method="post">
|
||||
<div class="mb-3 m-3 row">
|
||||
|
|
@ -246,7 +268,7 @@
|
|||
-->
|
||||
{{ range .Group.Filters }}
|
||||
<div class="modal fade" id="updateFilterModal_{{ .ID }}" tabindex="-1" aria-labelledby="updateFilterModalLabel_{{ .ID }}" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="updateFilterModalLabel_{{ .ID }}">Update Filter: {{ .Name }}</h5>
|
||||
|
|
@ -257,6 +279,9 @@
|
|||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link {{ if eq .Type "xpath" }} active {{ end }}" id="xpath-tab-{{ .ID }}" data-bs-toggle="tab" data-bs-target="#xpath-tab-pane-{{ .ID }}" aria-controls="xpath-tab-pane-{{ .ID }}" type="button" role="tab" aria-selected="{{ if eq .Type "xpath" }} true {{ else }} false {{ end }}">XPath</button>
|
||||
</li>
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link {{ if eq .Type "json" }} active {{ end }}" id="json-tab-{{ .ID }}" data-bs-toggle="tab" data-bs-target="#json-tab-pane-{{ .ID }}" aria-controls="json-tab-pane-{{ .ID }}" type="button" role="tab" aria-selected="{{ if eq .Type "json" }} true {{ else }} false {{ end }}">JSON</button>
|
||||
</li>
|
||||
<li class="nav-item" role="presentation">
|
||||
<button class="nav-link {{ if eq .Type "css" }} active {{ end }}" id="css-tab-{{ .ID }}" data-bs-toggle="tab" data-bs-target="#css-tab-pane-{{ .ID }}" aria-controls="css-tab-pane-{{ .ID }}" type="button" role="tab" aria-selected="{{ if eq .Type "css" }} true {{ else }} false {{ end }}">CSS</button>
|
||||
</li>
|
||||
|
|
@ -291,6 +316,26 @@
|
|||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="tab-pane fade {{ if eq .Type "json" }} show active {{ end }}" id="json-tab-pane-{{ .ID }}" role="tabpanel" aria-labelledby="json-tab-{{ .ID }}" tabindex="0">
|
||||
<form action="/filter/update" method="post">
|
||||
<div class="mb-3 m-3 row">
|
||||
<input type="hidden" name="filter_id" value="{{ .ID }}" >
|
||||
<input type="hidden" name="filter_group_id" value="{{ .FilterGroupID }}" >
|
||||
<input type="hidden" name="filter_type" value="json" >
|
||||
<input type="hidden" name="to" value="-" >
|
||||
|
||||
<label for="nameInput" class="col-sm-2 col-form-label">Name:</label>
|
||||
<div class="col-sm-10 p-2">
|
||||
<input type="text" class="form-control" name="name" id="nameInput" value="{{ .Name }}" placeholder="Name">
|
||||
</div>
|
||||
<label for="fromInput" class="col-sm-2 col-form-label">JSON:</label>
|
||||
<div class="col-sm-10 p-2">
|
||||
<input type="text" class="form-control" name="from" id="fromInput" value="{{ if eq .Type "json" }}{{ .From }}{{ end }}" placeholder="products.#.price']">
|
||||
</div>
|
||||
<button class="btn btn-primary mt-4">Update</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="tab-pane fade {{ if eq .Type "css" }} show active {{ end }}" id="css-tab-pane-{{ .ID }}" role="tabpanel" aria-labelledby="css-tab-{{ .ID }}" tabindex="0">
|
||||
<form action="/filter/update" method="post">
|
||||
<div class="mb-3 m-3 row">
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue