Skip to content

Commit fe66f9a

Browse files
gustavo-iniguez-goyagustavo-iniguez-goya
committed
rules: improved operator list parsing and conversion
Previously when creating a new rule we followed these steps: - Create a new protobuf Rule object from the ruleseditor or the pop-ups. - If the rule contained more than one operator, we converted the list of operators to a JSON string. - This JSON string was sent back to the daemon, and saved to the DB. - The list of operators were never expanded on the GUI, i.e., they were not saved as a list of protobuf Operator objects. - Once received in the daemon, the JSON string was parsed and converted to a protobuf Operator list of objects. Both, the JSON string and the list of protobuf Operator objects were saved to disk, but the JSON string was ignored when loading the rules. Saving the list of operators as a JSON string was a problem if you wanted to create or modify rules without the GUI. Now when creating or modifying rules from the GUI, the list of operators is no longer converted to JSON string. Instead the list is sent to the daemon as a list of protobuf Operators, and saved as JSON objects. Notes: - The JSON string is no longer saved to disk as part of the rules. - The list of operators is still saved as JSON string to the DB. - About not enabled rules: Previously, not enabled rules only had the list of operators as JSON string, with the field list:[] empty. Now the list of operators is saved as JSON objects, but if the rule is not enabled, it won't be parsed/loaded. Closes #1047 (cherry picked from commit b930510)
1 parent 4e7f1ae commit fe66f9a

12 files changed

Lines changed: 382 additions & 122 deletions

‎daemon/rule/loader.go‎

Lines changed: 118 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,70 @@ func (l *Loader) Load(path string) error {
9898
return nil
9999
}
100100

101+
// Add adds a rule to the list of rules, and optionally saves it to disk.
102+
func (l *Loader) Add(rule *Rule, saveToDisk bool) error {
103+
l.addUserRule(rule)
104+
if saveToDisk {
105+
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
106+
return l.Save(rule, fileName)
107+
}
108+
return nil
109+
}
110+
111+
// Replace adds a rule to the list of rules, and optionally saves it to disk.
112+
func (l *Loader) Replace(rule *Rule, saveToDisk bool) error {
113+
if err := l.replaceUserRule(rule); err != nil {
114+
return err
115+
}
116+
if saveToDisk {
117+
l.Lock()
118+
defer l.Unlock()
119+
120+
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
121+
return l.Save(rule, fileName)
122+
}
123+
return nil
124+
}
125+
126+
// Save a rule to disk.
127+
func (l *Loader) Save(rule *Rule, path string) error {
128+
rule.Updated = time.Now().Format(time.RFC3339)
129+
raw, err := json.MarshalIndent(rule, "", " ")
130+
if err != nil {
131+
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
132+
}
133+
134+
if err = ioutil.WriteFile(path, raw, 0600); err != nil {
135+
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
136+
}
137+
138+
return nil
139+
}
140+
141+
// Delete deletes a rule from the list by name.
142+
// If the duration is Always (i.e: saved on disk), it'll attempt to delete
143+
// it from disk.
144+
func (l *Loader) Delete(ruleName string) error {
145+
l.Lock()
146+
defer l.Unlock()
147+
148+
rule := l.rules[ruleName]
149+
if rule == nil {
150+
return nil
151+
}
152+
l.cleanListsRule(rule)
153+
154+
delete(l.rules, ruleName)
155+
l.sortRules()
156+
157+
if rule.Duration != Always {
158+
return nil
159+
}
160+
161+
log.Info("Delete() rule: %s", rule)
162+
return l.deleteRuleFromDisk(ruleName)
163+
}
164+
101165
func (l *Loader) loadRule(fileName string) error {
102166
raw, err := ioutil.ReadFile(fileName)
103167
if err != nil {
@@ -117,7 +181,13 @@ func (l *Loader) loadRule(fileName string) error {
117181
l.cleanListsRule(oldRule)
118182
}
119183

120-
if r.Enabled {
184+
if !r.Enabled {
185+
// XXX: we only parse and load the Data field if the rule is disabled and the Data field is not empty
186+
// the rule will remain disabled.
187+
if err = l.unmarshalOperatorList(&r.Operator); err != nil {
188+
return err
189+
}
190+
} else {
121191
if err := r.Operator.Compile(); err != nil {
122192
log.Warning("Operator.Compile() error: %s: %s", err, r.Operator.Data)
123193
return fmt.Errorf("(1) Error compiling rule: %s", err)
@@ -191,41 +261,6 @@ func (l *Loader) cleanListsRule(oldRule *Rule) {
191261
}
192262
}
193263

194-
func (l *Loader) liveReloadWorker() {
195-
l.liveReloadRunning = true
196-
197-
log.Debug("Rules watcher started on path %s ...", l.path)
198-
if err := l.watcher.Add(l.path); err != nil {
199-
log.Error("Could not watch path: %s", err)
200-
l.liveReloadRunning = false
201-
return
202-
}
203-
204-
for {
205-
select {
206-
case event := <-l.watcher.Events:
207-
// a new rule json file has been created or updated
208-
if event.Op&fsnotify.Write == fsnotify.Write {
209-
if strings.HasSuffix(event.Name, ".json") {
210-
log.Important("Ruleset changed due to %s, reloading ...", path.Base(event.Name))
211-
if err := l.loadRule(event.Name); err != nil {
212-
log.Warning("%s", err)
213-
}
214-
}
215-
} else if event.Op&fsnotify.Remove == fsnotify.Remove {
216-
if strings.HasSuffix(event.Name, ".json") {
217-
log.Important("Rule deleted %s", path.Base(event.Name))
218-
// we only need to delete from memory rules of type Always,
219-
// because the Remove event is of a file, i.e.: Duration == Always
220-
l.deleteRule(event.Name)
221-
}
222-
}
223-
case err := <-l.watcher.Errors:
224-
log.Error("File system watcher error: %s", err)
225-
}
226-
}
227-
}
228-
229264
func (l *Loader) isTemporary(r *Rule) bool {
230265
return r.Duration != Restart && r.Duration != Always && r.Duration != Once
231266
}
@@ -247,6 +282,18 @@ func (l *Loader) setUniqueName(rule *Rule) {
247282
}
248283
}
249284

285+
// Deprecated: rule.Operator.Data no longer holds the operator list in json format as string.
286+
func (l *Loader) unmarshalOperatorList(op *Operator) error {
287+
if op.Type == List && len(op.List) == 0 && op.Data != "" {
288+
if err := json.Unmarshal([]byte(op.Data), &op.List); err != nil {
289+
return fmt.Errorf("error loading rule of type list: %s", err)
290+
}
291+
op.Data = ""
292+
}
293+
294+
return nil
295+
}
296+
250297
func (l *Loader) sortRules() {
251298
l.rulesKeys = make([]string, 0, len(l.rules))
252299
for k := range l.rules {
@@ -278,22 +325,21 @@ func (l *Loader) replaceUserRule(rule *Rule) (err error) {
278325
l.cleanListsRule(oldRule)
279326
}
280327

328+
if err := l.unmarshalOperatorList(&rule.Operator); err != nil {
329+
log.Error(err.Error())
330+
}
331+
281332
if rule.Enabled {
282333
if err := rule.Operator.Compile(); err != nil {
283334
log.Warning("Operator.Compile() error: %s: %s", err, rule.Operator.Data)
284-
return fmt.Errorf("(2) Error compiling rule: %s", err)
335+
return fmt.Errorf("(2) error compiling rule: %s", err)
285336
}
286337

287338
if rule.Operator.Type == List {
288-
// TODO: use List protobuf object instead of un/marshalling to/from json
289-
if err = json.Unmarshal([]byte(rule.Operator.Data), &rule.Operator.List); err != nil {
290-
return fmt.Errorf("Error loading rule of type list: %s", err)
291-
}
292-
293339
for i := 0; i < len(rule.Operator.List); i++ {
294340
if err := rule.Operator.List[i].Compile(); err != nil {
295341
log.Warning("Operator.Compile() error: %s: ", err)
296-
return fmt.Errorf("(2) Error compiling list rule: %s", err)
342+
return fmt.Errorf("(2) error compiling list rule: %s", err)
297343
}
298344
}
299345
}
@@ -333,70 +379,39 @@ func (l *Loader) scheduleTemporaryRule(rule Rule) error {
333379
return nil
334380
}
335381

336-
// Add adds a rule to the list of rules, and optionally saves it to disk.
337-
func (l *Loader) Add(rule *Rule, saveToDisk bool) error {
338-
l.addUserRule(rule)
339-
if saveToDisk {
340-
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
341-
return l.Save(rule, fileName)
342-
}
343-
return nil
344-
}
345-
346-
// Replace adds a rule to the list of rules, and optionally saves it to disk.
347-
func (l *Loader) Replace(rule *Rule, saveToDisk bool) error {
348-
if err := l.replaceUserRule(rule); err != nil {
349-
return err
350-
}
351-
if saveToDisk {
352-
l.Lock()
353-
defer l.Unlock()
354-
355-
fileName := filepath.Join(l.path, fmt.Sprintf("%s.json", rule.Name))
356-
return l.Save(rule, fileName)
357-
}
358-
return nil
359-
}
360-
361-
// Save a rule to disk.
362-
func (l *Loader) Save(rule *Rule, path string) error {
363-
// When saving the rule, use always RFC3339 format for the Created field (#1140).
364-
rule.Updated = time.Now().Format(time.RFC3339)
365-
366-
raw, err := json.MarshalIndent(rule, "", " ")
367-
if err != nil {
368-
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
369-
}
370-
371-
if err = ioutil.WriteFile(path, raw, 0600); err != nil {
372-
return fmt.Errorf("Error while saving rule %s to %s: %s", rule, path, err)
373-
}
374-
375-
return nil
376-
}
377-
378-
// Delete deletes a rule from the list by name.
379-
// If the duration is Always (i.e: saved on disk), it'll attempt to delete
380-
// it from disk.
381-
func (l *Loader) Delete(ruleName string) error {
382-
l.Lock()
383-
defer l.Unlock()
382+
func (l *Loader) liveReloadWorker() {
383+
l.liveReloadRunning = true
384384

385-
rule := l.rules[ruleName]
386-
if rule == nil {
387-
return nil
385+
log.Debug("Rules watcher started on path %s ...", l.path)
386+
if err := l.watcher.Add(l.path); err != nil {
387+
log.Error("Could not watch path: %s", err)
388+
l.liveReloadRunning = false
389+
return
388390
}
389-
l.cleanListsRule(rule)
390391

391-
delete(l.rules, ruleName)
392-
l.sortRules()
393-
394-
if rule.Duration != Always {
395-
return nil
392+
for {
393+
select {
394+
case event := <-l.watcher.Events:
395+
// a new rule json file has been created or updated
396+
if event.Op&fsnotify.Write == fsnotify.Write {
397+
if strings.HasSuffix(event.Name, ".json") {
398+
log.Important("Ruleset changed due to %s, reloading ...", path.Base(event.Name))
399+
if err := l.loadRule(event.Name); err != nil {
400+
log.Warning("%s", err)
401+
}
402+
}
403+
} else if event.Op&fsnotify.Remove == fsnotify.Remove {
404+
if strings.HasSuffix(event.Name, ".json") {
405+
log.Important("Rule deleted %s", path.Base(event.Name))
406+
// we only need to delete from memory rules of type Always,
407+
// because the Remove event is of a file, i.e.: Duration == Always
408+
l.deleteRule(event.Name)
409+
}
410+
}
411+
case err := <-l.watcher.Errors:
412+
log.Error("File system watcher error: %s", err)
413+
}
396414
}
397-
398-
log.Info("Delete() rule: %s", rule)
399-
return l.deleteRuleFromDisk(ruleName)
400415
}
401416

402417
// FindFirstMatch will try match the connection against the existing rule set.

‎daemon/rule/loader_test.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package rule
22

33
import (
4+
"fmt"
45
"io"
56
"math/rand"
67
"os"
@@ -95,6 +96,57 @@ func TestRuleLoaderInvalidRegexp(t *testing.T) {
9596
})
9697
}
9798

99+
// Test rules of type operator.list. There're these scenarios:
100+
// - Enabled rules:
101+
// * operator Data field is ignored if it contains the list of operators as json string.
102+
// * the operarots list is expanded as json objecs under "list": []
103+
// For new rules (> v1.6.3), Data field will be empty.
104+
//
105+
// - Disabled rules
106+
// * (old) the Data field contains the list of operators as json string, and the list of operarots is empty.
107+
// * Data field empty, and the list of operators expanded.
108+
// In all cases the list of operators must be loaded.
109+
func TestRuleLoaderList(t *testing.T) {
110+
l, err := NewLoader(true)
111+
if err != nil {
112+
t.Fail()
113+
}
114+
115+
testRules := map[string]string{
116+
"rule-with-operator-list": "testdata/rule-operator-list.json",
117+
"rule-disabled-with-operators-list-as-json-string": "testdata/rule-disabled-operator-list.json",
118+
"rule-disabled-with-operators-list-expanded": "testdata/rule-disabled-operator-list-expanded.json",
119+
"rule-with-operator-list-data-empty": "testdata/rule-operator-list-data-empty.json",
120+
}
121+
122+
for name, path := range testRules {
123+
t.Run(fmt.Sprint("loadRule() ", path), func(t *testing.T) {
124+
if err := l.loadRule(path); err != nil {
125+
t.Error(fmt.Sprint("loadRule() ", path, " error:"), err)
126+
}
127+
t.Log("Test: List rule:", name, path)
128+
r, found := l.rules[name]
129+
if !found {
130+
t.Error(fmt.Sprint("loadRule() ", path, " not in the list:"), l.rules)
131+
}
132+
// Starting from > v1.6.3, after loading a rule of type List, the field Operator.Data is emptied, if the Data contained the list of operators as json.
133+
if len(r.Operator.List) != 2 {
134+
t.Error(fmt.Sprint("loadRule() ", path, " operator List not loaded:"), r)
135+
}
136+
if r.Operator.List[0].Type != Simple ||
137+
r.Operator.List[0].Operand != OpProcessPath ||
138+
r.Operator.List[0].Data != "/usr/bin/telnet" {
139+
t.Error(fmt.Sprint("loadRule() ", path, " operator List 0 not loaded:"), r)
140+
}
141+
if r.Operator.List[1].Type != Simple ||
142+
r.Operator.List[1].Operand != OpDstPort ||
143+
r.Operator.List[1].Data != "53" {
144+
t.Error(fmt.Sprint("loadRule() ", path, " operator List 1 not loaded:"), r)
145+
}
146+
})
147+
}
148+
}
149+
98150
func TestLiveReload(t *testing.T) {
99151
t.Parallel()
100152
t.Log("Test rules loader with live reload")

0 commit comments

Comments
 (0)