working 'store' filter

main.go

@@ -72,13 +72,17 @@ func (web Web) watchView(c *gin.Context) {
 	var connections []FilterConnection
 	web.db.Model(&FilterConnection{}).Where("watch_id = ?", watch.ID).Find(&connections)
 
+	var values []FilterOutput
+	web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values)
+
 	buildFilterTree(filters, connections)
-	fillFilterResults(filters)
+	processFilters(filters, web.db)
 
 	c.HTML(http.StatusOK, "watchView", gin.H{
 		"Watch":       watch,
 		"Filters":     filters,
 		"Connections": connections,
+		"Values":      values,
 	})
 }
 
@@ -219,7 +223,7 @@ func main() {
 	}
 
 	db, _ := gorm.Open(sqlite.Open(viper.GetString("database.dsn")))
-	db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{})
+	db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{})
 
 	//bot, _ := tgbotapi.NewBotAPI(viper.GetString("telegram.token"))
 

models.go

@@ -1,5 +1,7 @@
 package main
 
+import "time"
+
 type Watch struct {
 	ID       uint   `form:"watch_id" yaml:"watch_id"`
 	Name     string `form:"watch_name" yaml:"watch_name" binding:"required" validate:"min=1"`
@@ -12,7 +14,7 @@ type Filter struct {
 	Name     string    `form:"filter_name" yaml:"filter_name" json:"filter_name" binding:"required" validate:"min=1"`
 	X        int       `form:"x" yaml:"x" json:"x" validate:"default=0"`
 	Y        int       `form:"y" yaml:"y" json:"y" validate:"default=0"`
-	Type     string    `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math"`
+	Type     string    `form:"filter_type" yaml:"filter_type" json:"filter_type" binding:"required" validate:"oneof=url xpath json css replace match substring math store"`
 	Var1     string    `form:"var1" yaml:"var1" json:"var1" binding:"required"`
 	Var2     *string   `form:"var2" yaml:"var2" json:"var2"`
 	Var3     *string   `form:"var3" yaml:"var3" json:"var3"`
@@ -27,3 +29,11 @@ type FilterConnection struct {
 	OutputID uint `form:"filter_output_id" yaml:"filter_output_id" json:"filter_output_id" binding:"required"`
 	InputID  uint `form:"filter_input_id" yaml:"filter_input_id" json:"filter_input_id" binding:"required"`
 }
+
+type FilterOutput struct {
+	ID      uint      `yaml:"filter_output_id" json:"filter_output_id"`
+	WatchID uint      `yaml:"filter_output_watch_id" json:"filter_output_watch_id"`
+	Name    string    `yaml:"filter_output_name" json:"filter_output_name"`
+	Value   string    `yaml:"filter_output_value" json:"filter_output_value"`
+	Time    time.Time `yaml:"filter_output_time" json:"filter_output_time"`
+}

scraping.go

@@ -10,14 +10,16 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/andybalholm/cascadia"
 	"github.com/antchfx/htmlquery"
 	"github.com/tidwall/gjson"
 	"golang.org/x/net/html"
+	"gorm.io/gorm"
 )
 
-func fillFilterResults(filters []Filter) {
+func processFilters(filters []Filter, db *gorm.DB) {
 	processedMap := make(map[uint]bool, len(filters))
 	for len(filters) > 0 {
 		filter := &filters[0]
@@ -33,12 +35,12 @@ func fillFilterResults(filters []Filter) {
 			filters = append(filters, *filter)
 			continue
 		}
-		getFilterResult(filter)
+		getFilterResult(filter, db)
 		processedMap[filter.ID] = true
 	}
 }
 
-func getFilterResult(filter *Filter) {
+func getFilterResult(filter *Filter, db *gorm.DB) {
 	switch {
 	case filter.Type == "gurl":
 		{
@@ -101,7 +103,10 @@ func getFilterResult(filter *Filter) {
 				}
 			}
 		}
-
+	case filter.Type == "store":
+		{
+			storeFilterResult(filter, db)
+		}
 	default:
 		log.Println("getFilterResult called with filter.Type == ", filter.Type)
 	}
@@ -386,3 +391,24 @@ func getFilterResultRound(filter *Filter) {
 		}
 	}
 }
+
+func storeFilterResult(filter *Filter, db *gorm.DB) {
+	var previousOutput FilterOutput
+	db.Model(&FilterOutput{}).Order("time desc").Limit(1).Find(&previousOutput, "watch_id = ? AND name = ?", filter.WatchID, filter.Name)
+	for _, parent := range filter.Parents {
+		for _, result := range parent.Results {
+			if previousOutput.WatchID == 0 {
+				previousOutput.Name = filter.Name
+				previousOutput.Time = time.Now()
+				previousOutput.Value = result
+				previousOutput.WatchID = filter.WatchID
+				db.Create(&previousOutput)
+			} else {
+				previousOutput.Time = time.Now()
+				previousOutput.ID = 0
+				previousOutput.Value = result
+				db.Create(&previousOutput)
+			}
+		}
+	}
+}

static/edit.js

@@ -341,6 +341,33 @@ function onTypeChange(node) {
             var3Div.appendChild(var3Input);
             break;
         }
+        case "store": {
+            var var1Input = document.createElement("input");
+            var1Input.name = "var1";
+            var1Input.id = "var1Input";
+            var1Input.value = var1Value;
+            var1Input.classList.add("form-control");
+            var1Input.disabled = true;
+            var1Label.innerHTML = "-";
+            var1Input.placeholder = "";
+            var1Div.appendChild(var1Input);
+            var var2Input = document.createElement("input");
+            var2Input.name = "var2";
+            var2Input.id = "var2Input";
+            var2Input.value = var2Value;
+            var2Input.classList.add("form-control");
+            var2Input.disabled = true;
+            var2Label.innerHTML = "-";
+            var2Div.appendChild(var2Input);
+            var var3Input = document.createElement("input");
+            var3Input.name = "var3";
+            var3Input.id = "var3Input";
+            var3Input.value = var3Value;
+            var3Input.classList.add("form-control");
+            var3Input.disabled = true;
+            var3Label.innerHTML = "-";
+            var3Div.appendChild(var3Input);
+        }
     }
 }
 function onMathChange(node) {

static/edit.ts

@@ -330,6 +330,35 @@ function onTypeChange(node: DiagramNode | null = null){
             var3Div.appendChild(var3Input);
             break;
         }
+        case "store": {
+            let var1Input = document.createElement("input");
+            var1Input.name = "var1";
+            var1Input.id = "var1Input";
+            var1Input.value = var1Value;
+            var1Input.classList.add("form-control")
+            var1Input.disabled = true;
+            var1Label.innerHTML = "-";
+            var1Input.placeholder = "";
+            var1Div.appendChild(var1Input);
+
+            let var2Input = document.createElement("input");
+            var2Input.name = "var2";
+            var2Input.id = "var2Input";
+            var2Input.value = var2Value;
+            var2Input.classList.add("form-control")
+            var2Input.disabled = true;
+            var2Label.innerHTML = "-";
+            var2Div.appendChild(var2Input);
+
+            let var3Input = document.createElement("input");
+            var3Input.name = "var3";
+            var3Input.id = "var3Input";
+            var3Input.value = var3Value;
+            var3Input.classList.add("form-control");
+            var3Input.disabled = true;
+            var3Label.innerHTML = "-";
+            var3Div.appendChild(var3Input);
+        }
     }
 }
 

templates/watch/view.html

@@ -45,6 +45,7 @@
                                     <option value="match">Match</option>
                                     <option value="substring">Substring</option>
                                     <option value="math">Math</option>
+                                    <option value="store">Store</option>
                                 </select>
                             </div>
 
@@ -141,7 +142,7 @@ function canvasInit() {
             {{ .ID }}, 
             {{ .X }}, 
             {{ .Y }}, 
-            {{ .Name }}, 
+            "{{ .Name }}", 
             {
                 type: "{{ .Type }}",
                 var1: "{{ .Var1 }}",
@@ -154,8 +155,15 @@ function canvasInit() {
         diagrams.addConnectionById({{ .OutputID }}, {{ .InputID }});
     {{ end }}
     diagrams.fillParent();
+
+
+    {{ range .Values }}
+        console.log("{{ .Name }}", "{{ .Value }}", "{{ .Time }}")
+    {{ end }}
 }
 document.addEventListener('DOMContentLoaded', canvasInit, false);
 
+
+
 </script>
 {{ end }}