broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working 'store' filter

Browse source
This commit is contained in:
BroodjeAap 2022-09-30 10:59:15 +00:00
parent a75cdaf916
commit 5a0e6c0e43
6 changed files with 112 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
main.go
View file

@ -72,13 +72,17 @@ func (web Web) watchView(c *gin.Context) {
var connections []FilterConnection
web.db.Model(&FilterConnection{}).Where("watch_id = ?", watch.ID).Find(&connections)
var values []FilterOutput
web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values)
buildFilterTree(filters, connections)
fillFilterResults(filters)
processFilters(filters, web.db)
c.HTML(http.StatusOK, "watchView", gin.H{
"Watch": watch,
"Filters": filters,
"Connections": connections,
"Values": values,
})
}
@ -219,7 +223,7 @@ func main() {
}
db, _ := gorm.Open(sqlite.Open(viper.GetString("database.dsn")))
db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{})
db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{})
//bot, _ := tgbotapi.NewBotAPI(viper.GetString("telegram.token"))

12
models.go
View file

@ -1,5 +1,7 @@
package main
import "time"
type Watch struct {
ID uint `form:"watch_id" yaml:"watch_id"`
Name string `form:"watch_name" yaml:"watch_name" binding:"required" validate:"min=1"`
@ -12,7 +14,7 @@ type Filter struct {
Name string `form:"filter_name" yaml:"filter_name" json:"filter_name" binding:"required" validate:"min=1"`
X int `form:"x" yaml:"x" json:"x" validate:"default=0"`
Y int `form:"y" yaml:"y" json:"y" validate:"default=0"`
Type string `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math"`
Type string `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math store"`
Var1 string `form:"var1" yaml:"var1" json:"var1" binding:"required"`
Var2 *string `form:"var2" yaml:"var2" json:"var2"`
Var3 *string `form:"var3" yaml:"var3" json:"var3"`
@ -27,3 +29,11 @@ type FilterConnection struct {
OutputID uint `form:"filter_output_id" yaml:"filter_output_id" json:"filter_output_id" binding:"required"`
InputID uint `form:"filter_input_id" yaml:"filter_input_id" json:"filter_input_id" binding:"required"`
}
type FilterOutput struct {
ID uint `yaml:"filter_output_id" json:"filter_output_id"`
WatchID uint `yaml:"filter_output_watch_id" json:"filter_output_watch_id"`
Name string `yaml:"filter_output_name" json:"filter_output_name"`
Value string `yaml:"filter_output_value" json:"filter_output_value"`
Time time.Time `yaml:"filter_output_time" json:"filter_output_time"`
}

34
scraping.go
View file

@ -10,14 +10,16 @@ import (
"regexp"
"strconv"
"strings"
"time"
"github.com/andybalholm/cascadia"
"github.com/antchfx/htmlquery"
"github.com/tidwall/gjson"
"golang.org/x/net/html"
"gorm.io/gorm"
)
func fillFilterResults(filters []Filter) {
func processFilters(filters []Filter, db *gorm.DB) {
processedMap := make(map[uint]bool, len(filters))
for len(filters) > 0 {
filter := &filters[0]
@ -33,12 +35,12 @@ func fillFilterResults(filters []Filter) {
filters = append(filters, *filter)
continue
}
getFilterResult(filter)
getFilterResult(filter, db)
processedMap[filter.ID] = true
}
}
func getFilterResult(filter *Filter) {
func getFilterResult(filter *Filter, db *gorm.DB) {
switch {
case filter.Type == "gurl":
{
@ -101,7 +103,10 @@ func getFilterResult(filter *Filter) {
}
}
}
case filter.Type == "store":
{
storeFilterResult(filter, db)
}
default:
log.Println("getFilterResult called with filter.Type == ", filter.Type)
}
@ -386,3 +391,24 @@ func getFilterResultRound(filter *Filter) {
}
}
}
func storeFilterResult(filter *Filter, db *gorm.DB) {
var previousOutput FilterOutput
db.Model(&FilterOutput{}).Order("time desc").Limit(1).Find(&previousOutput, "watch_id = ? AND name = ?", filter.WatchID, filter.Name)
for _, parent := range filter.Parents {
for _, result := range parent.Results {
if previousOutput.WatchID == 0 {
previousOutput.Name = filter.Name
previousOutput.Time = time.Now()
previousOutput.Value = result
previousOutput.WatchID = filter.WatchID
db.Create(&previousOutput)
} else {
previousOutput.Time = time.Now()
previousOutput.ID = 0
previousOutput.Value = result
db.Create(&previousOutput)
}
}
}
}

27
static/edit.js
View file

@ -341,6 +341,33 @@ function onTypeChange(node) {
var3Div.appendChild(var3Input);
break;
}
case "store": {
var var1Input = document.createElement("input");
var1Input.name = "var1";
var1Input.id = "var1Input";
var1Input.value = var1Value;
var1Input.classList.add("form-control");
var1Input.disabled = true;
var1Label.innerHTML = "-";
var1Input.placeholder = "";
var1Div.appendChild(var1Input);
var var2Input = document.createElement("input");
var2Input.name = "var2";
var2Input.id = "var2Input";
var2Input.value = var2Value;
var2Input.classList.add("form-control");
var2Input.disabled = true;
var2Label.innerHTML = "-";
var2Div.appendChild(var2Input);
var var3Input = document.createElement("input");
var3Input.name = "var3";
var3Input.id = "var3Input";
var3Input.value = var3Value;
var3Input.classList.add("form-control");
var3Input.disabled = true;
var3Label.innerHTML = "-";
var3Div.appendChild(var3Input);
}
}
}
function onMathChange(node) {

29
static/edit.ts
View file

@ -330,6 +330,35 @@ function onTypeChange(node: DiagramNode | null = null){
var3Div.appendChild(var3Input);
break;
}
case "store": {
let var1Input = document.createElement("input");
var1Input.name = "var1";
var1Input.id = "var1Input";
var1Input.value = var1Value;
var1Input.classList.add("form-control")
var1Input.disabled = true;
var1Label.innerHTML = "-";
var1Input.placeholder = "";
var1Div.appendChild(var1Input);
let var2Input = document.createElement("input");
var2Input.name = "var2";
var2Input.id = "var2Input";
var2Input.value = var2Value;
var2Input.classList.add("form-control")
var2Input.disabled = true;
var2Label.innerHTML = "-";
var2Div.appendChild(var2Input);
let var3Input = document.createElement("input");
var3Input.name = "var3";
var3Input.id = "var3Input";
var3Input.value = var3Value;
var3Input.classList.add("form-control");
var3Input.disabled = true;
var3Label.innerHTML = "-";
var3Div.appendChild(var3Input);
}
}
}

10
templates/watch/view.html
View file

@ -45,6 +45,7 @@
<option value="match">Match</option>
<option value="substring">Substring</option>
<option value="math">Math</option>
<option value="store">Store</option>
</select>
</div>
@ -141,7 +142,7 @@ function canvasInit() {
{{ .ID }},
{{ .X }},
{{ .Y }},
{{ .Name }},
"{{ .Name }}",
{
type: "{{ .Type }}",
var1: "{{ .Var1 }}",
@ -154,8 +155,15 @@ function canvasInit() {
diagrams.addConnectionById({{ .OutputID }}, {{ .InputID }});
{{ end }}
diagrams.fillParent();
{{ range .Values }}
console.log("{{ .Name }}", "{{ .Value }}", "{{ .Time }}")
{{ end }}
}
document.addEventListener('DOMContentLoaded', canvasInit, false);
</script>
{{ end }}