From cec960e6ddc862ecde5a6d89af87f8e9c5afba9e Mon Sep 17 00:00:00 2001 From: BroodjeAap Date: Sat, 18 Mar 2023 12:36:09 +0000 Subject: [PATCH] added 'expect' filter --- README.md | 6 + models/expect.go | 10 + todo.md | 2 - web/scraping.go | 51 +++++ web/scraping_test.go | 395 +++++++++++++++++++++++++++++++++- web/static/edit.js | 38 +++- web/static/edit.ts | 25 +++ web/templates/watch/edit.html | 1 + web/web.go | 4 +- 9 files changed, 523 insertions(+), 9 deletions(-) create mode 100644 models/expect.go diff --git a/README.md b/README.md index a1dcd74..3706efb 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Some out-of-the-box highlights: - [Substring](#substring) - [Contains](#contains) - [Store](#store) + - [Expect](#expect) - [Notify](#notify) - [Math](#math) - [Sum](#sum) @@ -445,6 +446,11 @@ Inputs pass if they contain the given regex. Stores each input value in the database under its own name. It's recommended to do this after reducing inputs to a single value (Minimum/Maximum/Average/etc). +## Expect + +Outputs a value when it has no inputs, useful to do something (notify) when something goes wrong with your Watch. +Will only trigger once and can be set to wait multiple times before triggering. + ## Notify Executes the given template and sends the resulting string as a message to the given notifier(s). diff --git a/models/expect.go b/models/expect.go new file mode 100644 index 0000000..c653001 --- /dev/null +++ b/models/expect.go @@ -0,0 +1,10 @@ +package models + +import "time" + +type ExpectFail struct { + ID uint `yaml:"expect_fail_id" json:"expect_fail_id"` + WatchID uint `yaml:"expect_fail_watch_id" gorm:"index" json:"expect_fail_watch_id"` + Name string `yaml:"expect_fail_name" json:"expect_fail_name"` + Time time.Time `yaml:"expect_fail_time" json:"expect_fail_time"` +} diff --git a/todo.md b/todo.md index 436c081..5717ab3 100644 --- a/todo.md +++ b/todo.md @@ -1,5 +1,3 @@ # Todo -- add 'expect' filter, outputs on no inputs - - seperate 'expect' table in db, after x number of expects passing really pass - add time 'jitter' to schedule filter - var2 with another schedule string, var1 + (var2 * random) duration \ No newline at end of file diff --git a/web/scraping.go b/web/scraping.go index 8d01b03..4e3a827 100644 --- a/web/scraping.go +++ b/web/scraping.go @@ -230,6 +230,10 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, d { storeFilterResult(filter, web.db, debug) } + case filter.Type == "expect": + { + getFilterResultExpect(filter, web, debug) + } case filter.Type == "notify": { notifyFilter(filters, filter, watch, web, debug) @@ -1266,6 +1270,53 @@ func getFilterResultLua(filter *Filter) { ) } +// getFilterResultExpect outputs once if there is no results from its parents a set number of times +func getFilterResultExpect(filter *Filter, web *Web, debug bool) { + if len(filter.Parents) == 0 { + filter.Logs = append(filter.Logs, "Need Parents") + return + } + for i := range filter.Parents { + parent := filter.Parents[i] + if len(parent.Results) > 0 { // reset/delete expectFails + web.db.Delete(&ExpectFail{}, "watch_id = ? AND name = ?", filter.WatchID, filter.Name) + return + } + } + if debug { + filter.Results = append(filter.Results, "Expected") + return + } + expectThreshold, err := strconv.Atoi(filter.Var1) + if err != nil { + filter.Logs = append(filter.Logs, "Could not parse to int:", filter.Var1) + expectThreshold = 1 + } + if expectThreshold <= 0 { + // 0 doesn't really make sense so just set to 1 + expectThreshold = 1 + } + + var expectFails []ExpectFail + web.db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ? AND name = ?", filter.WatchID, filter.Name) + + // +1 so this one is already counted + failCount := len(expectFails) + 1 + + if failCount > expectThreshold { + return + } else if expectThreshold == failCount { + filter.Results = append(filter.Results, "Expected") + } + + expectFail := ExpectFail{ + WatchID: filter.WatchID, + Name: filter.Name, + Time: time.Now(), + } + web.db.Create(&expectFail) +} + // getFilterResultEcho is a debug filter type, used to bootstrap some tests func getFilterResultEcho(filter *Filter) { filter.Results = append(filter.Results, filter.Var1) diff --git a/web/scraping_test.go b/web/scraping_test.go index 49a6b1b..350ca31 100644 --- a/web/scraping_test.go +++ b/web/scraping_test.go @@ -699,7 +699,7 @@ func TestFilterRound(t *testing.T) { func getTestDB() *gorm.DB { db, _ := gorm.Open(sqlite.Open("./test.db")) - db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}) + db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}, &ExpectFail{}) return db } @@ -1765,8 +1765,6 @@ func TestSimpleTriggeredWatch(t *testing.T) { maxFilter := &filters[6] storeMaxFilter := &filters[7] - log.Println(scheduleFilter) - connections := []FilterConnection{ { WatchID: watch.ID, @@ -1806,8 +1804,6 @@ func TestSimpleTriggeredWatch(t *testing.T) { } db.Create(&connections) - log.Println(connections[0]) - TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) var filterOutputs []FilterOutput @@ -1874,3 +1870,392 @@ func TestDontAllowMultipleCronOnSingleFilter(t *testing.T) { t.Errorf("Expected error message in filter log, found empty log: %s", filter.Logs) } } + +func TestWatchWithExpectNotTriggering(t *testing.T) { + db := getTestDB() + filters := []Filter{ + { + ID: 0, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + ID: 1, + Name: "XPath", + Type: "xpath", + Var1: "//td[@class='price']", + }, + { + ID: 2, + Name: "Expect", + Type: "expect", + Var1: "1", + }, + } + + expectFilter := &filters[2] + + connections := []FilterConnection{ + { + OutputID: 0, + InputID: 1, + }, + { + OutputID: 1, + InputID: 2, + }, + } + + buildFilterTree(filters, connections) + ProcessFilters(filters, &Web{db: db}, nil, false, nil) + + if len(expectFilter.Results) != 0 { + t.Error("Expect has results, should be empty:", expectFilter.Results) + } + + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} + +func TestWatchWithExpectTriggering(t *testing.T) { + db := getTestDB() + filters := []Filter{ + { + ID: 0, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + ID: 1, + Name: "XPath", + Type: "xpath", + Var1: "//div[@class='price']", + }, + { + ID: 2, + Name: "Expect", + Type: "expect", + Var1: "1", + }, + } + + expectFilter := &filters[2] + + connections := []FilterConnection{ + { + OutputID: 0, + InputID: 1, + }, + { + OutputID: 1, + InputID: 2, + }, + } + + buildFilterTree(filters, connections) + ProcessFilters(filters, &Web{db: db}, nil, false, nil) + + if len(expectFilter.Results) != 1 { + t.Error("Expect has no results, should have 'expected'") + } + + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} +func TestWatchWithExpect3Triggering(t *testing.T) { + db := getTestDB() + filters := []Filter{ + { + ID: 0, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + ID: 1, + Name: "XPath", + Type: "xpath", + Var1: "//div[@class='price']", + }, + { + ID: 2, + Name: "Expect", + Type: "expect", + Var1: "3", + }, + } + + expectFilter := &filters[2] + + connections := []FilterConnection{ + { + OutputID: 0, + InputID: 1, + }, + { + OutputID: 1, + InputID: 2, + }, + } + + buildFilterTree(filters, connections) + + ProcessFilters(filters, &Web{db: db}, nil, false, nil) + + if len(expectFilter.Results) != 0 { + t.Error("Expect has results, should be empty:", expectFilter.Results) + } + + ProcessFilters(filters, &Web{db: db}, nil, false, nil) + + if len(expectFilter.Results) != 0 { + t.Error("Expect has results, should be empty:", expectFilter.Results) + } + + ProcessFilters(filters, &Web{db: db}, nil, false, nil) + + if len(expectFilter.Results) != 1 { + t.Error("Expect has no results, should have 'expected'") + } + + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} + +func TestWatchWithExpectNotTriggeringDB(t *testing.T) { + db := getTestDB() + watch := Watch{ + Name: "Test", + } + db.Create(&watch) + filters := []Filter{ + { + WatchID: watch.ID, + Name: "Schedule", + Type: "cron", + }, + { + WatchID: watch.ID, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + WatchID: watch.ID, + Name: "XPath", + Type: "xpath", + Var1: "//td[@class='price']", + }, + { + WatchID: watch.ID, + Name: "Expect", + Type: "expect", + Var1: "1", + }, + } + db.Create(&filters) + scheduleFilter := &filters[0] + echoFilter := &filters[1] + xpathFilter := &filters[2] + expectFilter := &filters[3] + + connections := []FilterConnection{ + { + WatchID: watch.ID, + OutputID: scheduleFilter.ID, + InputID: echoFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: echoFilter.ID, + InputID: xpathFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: xpathFilter.ID, + InputID: expectFilter.ID, + }, + } + db.Create(&connections) + + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + var expectFails []ExpectFail + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) > 0 { + t.Errorf("Found ExpectFail values expected none!") + } + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} +func TestWatchWithExpectTriggeringDB(t *testing.T) { + db := getTestDB() + watch := Watch{ + Name: "Test", + } + db.Create(&watch) + filters := []Filter{ + { + WatchID: watch.ID, + Name: "Schedule", + Type: "cron", + }, + { + WatchID: watch.ID, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + WatchID: watch.ID, + Name: "XPath", + Type: "xpath", + Var1: "//div[@class='price']", + }, + { + WatchID: watch.ID, + Name: "Expect", + Type: "expect", + Var1: "1", + }, + } + db.Create(&filters) + scheduleFilter := &filters[0] + echoFilter := &filters[1] + xpathFilter := &filters[2] + expectFilter := &filters[3] + + connections := []FilterConnection{ + { + WatchID: watch.ID, + OutputID: scheduleFilter.ID, + InputID: echoFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: echoFilter.ID, + InputID: xpathFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: xpathFilter.ID, + InputID: expectFilter.ID, + }, + } + db.Create(&connections) + + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + var expectFails []ExpectFail + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) != 1 { + t.Errorf("Found no ExpectFail values expected 1!") + } + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} +func TestWatchWithExpect3TriggeringDB(t *testing.T) { + db := getTestDB() + watch := Watch{ + Name: "Test", + } + db.Create(&watch) + filters := []Filter{ + { + WatchID: watch.ID, + Name: "Schedule", + Type: "cron", + }, + { + WatchID: watch.ID, + Name: "Echo", + Type: "echo", + Var1: HTML_STRING, + }, + { + WatchID: watch.ID, + Name: "XPath", + Type: "xpath", + Var1: "//div[@class='price']", + }, + { + WatchID: watch.ID, + Name: "Expect", + Type: "expect", + Var1: "3", + }, + } + db.Create(&filters) + scheduleFilter := &filters[0] + echoFilter := &filters[1] + xpathFilter := &filters[2] + expectFilter := &filters[3] + + connections := []FilterConnection{ + { + WatchID: watch.ID, + OutputID: scheduleFilter.ID, + InputID: echoFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: echoFilter.ID, + InputID: xpathFilter.ID, + }, + { + WatchID: watch.ID, + OutputID: xpathFilter.ID, + InputID: expectFilter.ID, + }, + } + db.Create(&connections) + + var expectFails []ExpectFail + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) != 1 { + t.Errorf("Found %d ExpectFail values, expected 1!", len(expectFails)) + log.Println(expectFails) + } + + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) != 2 { + t.Errorf("Found %d ExpectFail values, expected 2!", len(expectFails)) + log.Println(expectFails) + } + + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) != 3 { + t.Errorf("Found %d ExpectFail values, expected 3! (1)", len(expectFails)) + log.Println(expectFails) + } + TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID) + + db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID) + if len(expectFails) != 3 { + t.Errorf("Found %d ExpectFail values, expected 3! (2)", len(expectFails)) + log.Println(expectFails) + } + + err := os.Remove("./test.db") + if err != nil { + log.Println("Could not remove test db:", err) + } +} diff --git a/web/static/edit.js b/web/static/edit.js index 3ba6bfd..8694a7e 100644 --- a/web/static/edit.js +++ b/web/static/edit.js @@ -34,6 +34,8 @@ var urlPrefix = getURLPrefix(); function onTypeChange(node) { var e_1, _a, e_2, _b; if (node === void 0) { node = null; } + // onTypeChange handles changing of the type of a DiagramNode while editing or creating a new Node + // It removes all input elements and each case is responsible for adding the input it needs // @ts-ignore var urlPrefix = getURLPrefix(); var select = document.getElementById("typeInput"); @@ -415,6 +417,29 @@ function onTypeChange(node) { onConditionChange(node); break; } + case "expect": { + var var1Input = document.createElement("input"); + var1Input.name = "var1"; + var1Input.id = "var1Input"; + var1Input.type = "number"; + var1Input.value = "1"; + var1Input.classList.add("form-control"); + var1Label.innerHTML = "Threshold"; + var1Input.placeholder = "1"; + if (var1Value != "") { + var1Input.value = var1Value; + } + var1Div.appendChild(var1Input); + var var2Input = document.createElement("input"); + var2Input.name = "var2"; + var2Input.id = "var2Input"; + var2Input.value = var2Value; + var2Input.classList.add("form-control"); + var2Input.disabled = true; + var2Label.innerHTML = "-"; + var2Div.appendChild(var2Input); + break; + } case "notify": { var var1Input = document.createElement("textarea"); var1Input.name = "var1"; @@ -607,6 +632,7 @@ function onTypeChange(node) { } function onMathChange(node) { if (node === void 0) { node = null; } + // onMatchChange handles the changing of the inputs when type == math var var1Input = document.getElementById("var1Input"); var var1Label = document.getElementById("var1Label"); var var2Input = document.getElementById("var2Input"); @@ -633,6 +659,7 @@ function onMathChange(node) { function onConditionChange(node) { var e_3, _a; if (node === void 0) { node = null; } + // onConditionChange handles the changing of the inputs when type == condition var var1Input = document.getElementById("var1Input"); var var1Label = document.getElementById("var1Label"); var var1Div = document.getElementById("var1Div"); @@ -713,6 +740,7 @@ function onConditionChange(node) { } } function onBrowserlessChange(node) { + // onBrowserlessChange handles the changing of the inputs when type == browserless if (node === void 0) { node = null; } var var1Input = document.getElementById("var1Input"); var var1Label = document.getElementById("var1Label"); @@ -790,7 +818,7 @@ function onBrowserlessChange(node) { var var2Input_6 = document.createElement("textarea"); var2Input_6.name = "var2Input"; var2Input_6.id = "var2Input"; - var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};"; + var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n // click something\n //await page.click(\"#elem\");\n \n // fill input\n //await page.$eval('#elem', el => el.value = 'some text');\n \n // select dropdown\n // await page.select('#elem', 'value')\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};"; var2Input_6.classList.add("form-control"); var2Input_6.rows = 15; var2Label.innerHTML = "Code"; @@ -816,6 +844,7 @@ function onBrowserlessChange(node) { } } function onSubmitNewFilter() { + // onSubmitNewFilter collects all the values from the input elements, and calls _diagram.addNode() with it var nameInput = document.getElementById("nameInput"); var name = nameInput.value; var selectType = document.getElementById("typeInput"); @@ -830,6 +859,7 @@ function onSubmitNewFilter() { } function editNode(node) { var e_4, _a, e_5, _b; + // editNode resets the edit/new Node modal to reflect the values of 'node' var addFilterButton = document.getElementById("filterButton"); addFilterButton.click(); var name = node.label; @@ -919,6 +949,7 @@ function editNode(node) { submitButton.onclick = function () { submitEditNode(node); }; } function deleteNode(node) { + // deleteNode deletes a node from _diagram and removes all connections to/from it _diagram.nodes.delete(node.id); for (var i = 0; i < _diagram.connections.length; i++) { var connection = _diagram.connections[i]; @@ -931,6 +962,7 @@ function deleteNode(node) { } } function submitEditNode(node) { + // submitEditNode saves the changes to the input elements to the underlying node var nameInput = document.getElementById("nameInput"); node.label = nameInput.value; var selectType = document.getElementById("typeInput"); @@ -950,6 +982,7 @@ function submitEditNode(node) { } function saveWatch() { var e_6, _a, e_7, _b; + // saveWatch collects all the state (nodes/connections), turns it into JSON and submits it through a hidden form var watchIdInput = document.getElementById("watch_id"); var watchId = Number(watchIdInput.value); var filters = new Array(); @@ -1006,6 +1039,7 @@ function saveWatch() { saveWatchForm.submit(); } function addFilterButtonClicked() { + // addFilterButtonClicked opens up the new/edit filter modal and empties it var submitButton = document.getElementById("submitFilterButton"); submitButton.onclick = onSubmitNewFilter; submitButton.innerHTML = "Add Filter"; @@ -1016,6 +1050,7 @@ function addFilterButtonClicked() { onTypeChange(); } function pageInit() { + // pageInit sets all the onclick/onchange trigger events var select = document.getElementById("typeInput"); select.onchange = function () { onTypeChange(); }; var addFilterButton = document.getElementById("filterButton"); @@ -1027,6 +1062,7 @@ function pageInit() { } document.addEventListener('DOMContentLoaded', pageInit, false); function clearCache() { + // POSTs to cache/clear and reloads if clearing the cache was succesful var confirmed = confirm("Do you want to clear the URL cache?"); if (!confirmed) { return; // do nothing diff --git a/web/static/edit.ts b/web/static/edit.ts index a929413..fd415a6 100644 --- a/web/static/edit.ts +++ b/web/static/edit.ts @@ -398,6 +398,31 @@ function onTypeChange(node: DiagramNode | null = null){ onConditionChange(node); break; } + case "expect": { + let var1Input = document.createElement("input"); + var1Input.name = "var1"; + var1Input.id = "var1Input"; + var1Input.type = "number"; + var1Input.value = "1"; + var1Input.classList.add("form-control") + var1Label.innerHTML = "Threshold"; + var1Input.placeholder = "1"; + if (var1Value != ""){ + var1Input.value = var1Value; + } + var1Div.appendChild(var1Input); + + + let var2Input = document.createElement("input"); + var2Input.name = "var2"; + var2Input.id = "var2Input"; + var2Input.value = var2Value; + var2Input.classList.add("form-control") + var2Input.disabled = true; + var2Label.innerHTML = "-"; + var2Div.appendChild(var2Input); + break; + } case "notify":{ let var1Input = document.createElement("textarea"); var1Input.name = "var1"; diff --git a/web/templates/watch/edit.html b/web/templates/watch/edit.html index fc823e3..516a46c 100644 --- a/web/templates/watch/edit.html +++ b/web/templates/watch/edit.html @@ -88,6 +88,7 @@ GoWatch Edit {{ .Watch.Name }} + diff --git a/web/web.go b/web/web.go index 3047703..f33f249 100644 --- a/web/web.go +++ b/web/web.go @@ -165,7 +165,7 @@ func (web *Web) initDB() { } break } - web.db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}) + web.db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}, ExpectFail{}) } // initRouer initializes the GoWatch routes, binding web.func to a url path @@ -663,6 +663,7 @@ func (web *Web) deleteWatch(c *gin.Context) { web.db.Delete(&FilterConnection{}, "watch_id = ?", id) web.db.Delete(&FilterOutput{}, "watch_id = ?", id) + web.db.Delete(&ExpectFail{}, "watch_id = ?", id) var cronFilters []Filter web.db.Model(&Filter{}).Find(&cronFilters, "watch_id = ? AND type = 'cron' AND var2 = 'yes'", id) @@ -808,6 +809,7 @@ func (web *Web) watchUpdate(c *gin.Context) { } web.db.Delete(&Filter{}, "watch_id = ?", watch.ID) + web.db.Delete(&ExpectFail{}, "watch_id = ?", watch.ID) filterMap := make(map[uint]*Filter) if len(newFilters) > 0 {