Merge branch 'feature-auditlog' into develop

api/system/spec.json

@@ -1759,7 +1759,6 @@
       }
     ]
   },
-
   {
     "title": "Statistics",
     "entrypoint": "stats",
@@ -1863,5 +1862,63 @@
         }
       }
     ]
+  },
+  {
+    "title": "Action log",
+    "entrypoint": "actionlog",
+    "path": "/actionlog",
+    "struct": [
+      {
+        "imports": [
+          "time"
+        ]
+      }
+    ],
+    "apis": [
+      {
+        "name": "list",
+        "method": "GET",
+        "title": "Action log events",
+        "path": "/",
+        "parameters": {
+          "get": [
+            {
+              "name": "from",
+              "type": "*time.Time",
+              "required": false,
+              "title": "From"
+            },
+            {
+              "name": "to",
+              "type": "*time.Time",
+              "required": false,
+              "title": "To"
+            },
+            {
+              "name": "resource",
+              "required": false,
+              "title": "Resource",
+              "type": "string"
+            },
+            {
+              "name": "action",
+              "required": false,
+              "title": "Action",
+              "type": "string"
+            },
+            {
+              "name": "actorID",
+              "required": false,
+              "title": "Filter by one or more actors",
+              "type": "[]string"
+            },
+            {"type": "uint",   "name": "limit",   "title": "Limit"},
+            {"type": "uint",   "name": "offset",  "title": "Offset"},
+            {"type": "uint",   "name": "page",    "title": "Page number (1-based)"},
+            {"type": "uint",   "name": "perPage", "title": "Returned items per page (default 50)"}
+          ]
+        }
+      }
+    ]
   }
 ]

api/system/spec/actionlog.json

+{
+  "Title": "Action log",
+  "Interface": "Actionlog",
+  "Struct": [
+    {
+      "imports": [
+        "time"
+      ]
+    }
+  ],
+  "Parameters": null,
+  "Protocol": "",
+  "Authentication": null,
+  "Path": "/actionlog",
+  "APIs": [
+    {
+      "Name": "list",
+      "Method": "GET",
+      "Title": "Action log events",
+      "Path": "/",
+      "Parameters": {
+        "get": [
+          {
+            "name": "from",
+            "required": false,
+            "title": "From",
+            "type": "*time.Time"
+          },
+          {
+            "name": "to",
+            "required": false,
+            "title": "To",
+            "type": "*time.Time"
+          },
+          {
+            "name": "resource",
+            "required": false,
+            "title": "Resource",
+            "type": "string"
+          },
+          {
+            "name": "action",
+            "required": false,
+            "title": "Action",
+            "type": "string"
+          },
+          {
+            "name": "actorID",
+            "required": false,
+            "title": "Filter by one or more actors",
+            "type": "[]string"
+          },
+          {
+            "name": "limit",
+            "title": "Limit",
+            "type": "uint"
+          },
+          {
+            "name": "offset",
+            "title": "Offset",
+            "type": "uint"
+          },
+          {
+            "name": "page",
+            "title": "Page number (1-based)",
+            "type": "uint"
+          },
+          {
+            "name": "perPage",
+            "title": "Returned items per page (default 50)",
+            "type": "uint"
+          }
+        ]
+      }
+    }
+  ]
+}
\ No newline at end of file

codegen.sh

@@ -104,6 +104,9 @@ function types {
 	./build/gen-type-set --types Script --output pkg/corredor/types.gen.go --with-primary-key=false --package corredor
 	./build/gen-type-set-test --types Script --output pkg/corredor/types.gen_test.go --with-primary-key=false --package corredor
 
+	./build/gen-type-set --types Action --output pkg/actionlog/types.gen.go --with-primary-key=false --package actionlog
+	./build/gen-type-set-test --types Action --output pkg/actionlog/types.gen_test.go --with-primary-key=false --package actionlog
+
 
 	green "OK"
 }
@@ -133,7 +136,7 @@ function provision {
 
 function events {
   if [ ! -f "build/event-gen" ]; then
-		CGO_ENABLED=0 go build -o ./build/event-gen codegen/v2/events.go
+		CGO_ENABLED=0 go build -o ./build/event-gen ./codegen/v2/events
 	fi
 
 	for SERVICE in system compose messaging; do

codegen/v2/actionlog/actions.gen.go.tpl

+package {{ .Package }}
+
+// This file is auto-generated from {{ .YAML }}
+//
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"errors"
+	"time"
+{{- if $.SupportHttpErrors }}
+	"net/http"
+{{- end }}
+
+	"github.com/cortezaproject/corteza-server/pkg/actionlog"
+{{- range $import := $.Import }}
+    {{ $import }}
+{{- end }}
+)
+
+type (
+	{{ $.Service }}ActionProps struct {
+	{{- range $prop := $.Props }}
+		{{ $prop.Name }} {{ $prop.Type }}
+	{{- end }}
+	}
+
+{{ if $.Actions }}
+	{{ $.Service }}Action struct {
+		timestamp time.Time
+		resource  string
+		action    string
+		log       string
+		severity  actionlog.Severity
+
+		// prefix for error when action fails
+		errorMessage string
+
+		props *{{ $.Service }}ActionProps
+	}
+{{ end }}
+
+{{ if $.Errors }}
+	{{ $.Service }}Error struct {
+		timestamp time.Time
+		error       string
+		resource    string
+		action      string
+		message 	string
+		log         string
+		severity    actionlog.Severity
+
+		wrap        error
+
+		props       *{{ $.Service }}ActionProps
+
+    {{ if $.SupportHttpErrors }}
+        httpStatusCode int
+    {{ end }}
+	}
+{{ end }}
+)
+
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+// Props methods
+
+{{- range $prop := $.Props }}
+// {{ camelCase "set" $prop.Name }} updates {{ $.Service }}ActionProps's {{ $prop.Name }}
+//
+// Allows method chaining
+//
+// This function is auto-generated.
+//
+func (p *{{ $.Service }}ActionProps) {{ camelCase "set" $prop.Name }}({{ $prop.Name }} {{ $prop.Type }}) *{{ $.Service }}ActionProps {
+    p.{{ $prop.Name }} = {{ $prop.Name }}
+    return p
+}
+{{ end }}
+
+
+// serialize converts {{ $.Service }}ActionProps to actionlog.Meta
+//
+// This function is auto-generated.
+//
+func (p {{ $.Service }}ActionProps) serialize() actionlog.Meta {
+	var (
+		m   = make(actionlog.Meta)
+		str = func(i interface{}) string { return fmt.Sprintf("%v", i) }
+	)
+
+	// avoiding declared but not used
+	_ = str
+
+{{ range $prop := $.Props }}
+	{{- if $prop.Builtin }}
+	    m["{{ $prop.Name }}"] = str(p.{{ $prop.Name }})
+	{{- else }}
+	if p.{{ $prop.Name }} != nil {
+    {{- range $f := $prop.Fields }}
+        m["{{ $prop.Name }}.{{ $f }}"] = str(p.{{ $prop.Name }}.{{ camelCase " " $f }})
+    {{- end }}
+    }
+    {{- end }}
+{{- end }}
+
+    return m
+}
+
+// tr translates string and replaces meta value placeholder with values
+//
+// This function is auto-generated.
+//
+func (p {{ $.Service }}ActionProps) tr(in string, err error) string {
+    var (
+        pairs = []string{"{err}"}
+
+{{- if $.Props }}
+        // first non-empty string
+        fns = func(ii ... interface{}) string {
+			for _, i:= range ii {
+				if s :=fmt.Sprintf("%v", i); len(s) > 0 {
+					return s
+				}
+			}
+
+			return ""
+		}
+{{- end }}
+    )
+
+	if err != nil {
+		for {
+			// Unwrap errors
+			ue := errors.Unwrap(err)
+			if ue == nil {
+				break
+			}
+
+			err = ue
+		}
+
+	    pairs = append(pairs, err.Error())
+	} else {
+	    pairs = append(pairs, "nil")
+	}
+
+{{- range $prop := $.Props }}
+	{{- if $prop.Builtin }}
+	    pairs = append(pairs, "{{"{"}}{{ $prop.Name }}}", fns(p.{{ $prop.Name }}))
+	{{- else }}
+
+	if p.{{ $prop.Name }} != nil {
+	    // replacement for "{{"{"}}{{ $prop.Name }}}" (in order how fields are defined)
+        pairs = append(
+            pairs,
+            "{{"{"}}{{ $prop.Name }}}",
+            fns(
+            {{- range $f := $prop.Fields }}
+                p.{{ $prop.Name }}.{{ camelCase " " $f }},
+            {{- end }}
+            ),
+        )
+
+    {{- range $f := $prop.Fields }}
+        pairs = append(pairs, "{{"{"}}{{ $prop.Name }}.{{ $f }}}", fns(p.{{ $prop.Name }}.{{ camelCase " " $f }}))
+    {{- end }}
+    }
+    {{- end }}
+{{- end }}
+    return strings.NewReplacer(pairs...).Replace(in)
+}
+
+{{ if $.Actions }}
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+// Action methods
+
+// String returns loggable description as string
+//
+// This function is auto-generated.
+//
+func (a *{{ $.Service }}Action) String() string {
+    var props = &{{ $.Service }}ActionProps{}
+
+    if a.props != nil {
+        props = a.props
+    }
+
+    return props.tr(a.log, nil)
+}
+
+func (e *{{ $.Service }}Action) LoggableAction() *actionlog.Action {
+	return &actionlog.Action{
+		Timestamp:   e.timestamp,
+		Resource:    e.resource,
+		Action:      e.action,
+		Severity:    e.severity,
+		Description: e.String(),
+		Meta:        e.props.serialize(),
+	}
+}
+{{ end }}
+
+{{ if $.Errors }}
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+// Error methods
+
+// String returns loggable description as string
+//
+// It falls back to message if log is not set
+//
+// This function is auto-generated.
+//
+func (e *{{ $.Service }}Error) String() string {
+    var props = &{{ $.Service }}ActionProps{}
+
+    if e.props != nil {
+        props = e.props
+    }
+
+
+    if e.wrap != nil && !strings.Contains(e.log, "{err}") {
+        // Suffix error log with {err} to ensure
+        // we log the cause for this error
+        e.log += ": {err}"
+    }
+
+	return props.tr(e.log, e.wrap)
+}
+
+// Error satisfies
+//
+// This function is auto-generated.
+//
+func (e *{{ $.Service }}Error) Error() string {
+    var props = &{{ $.Service }}ActionProps{}
+
+    if e.props != nil {
+        props = e.props
+    }
+
+	return props.tr(e.message, e.wrap)
+}
+
+// Is fn for error equality check
+//
+// This function is auto-generated.
+//
+func (e *{{ $.Service }}Error) Is(Resource error) bool {
+	t, ok := Resource.(*{{ $.Service }}Error)
+	if !ok {
+		return false
+	}
+
+	return t.resource == e.resource && t.error == e.error
+}
+
+// Wrap wraps {{ $.Service }}Error around another error
+//
+// This function is auto-generated.
+//
+func (e *{{ $.Service }}Error) Wrap(err error) *{{ $.Service }}Error {
+    e.wrap = err
+    return e
+}
+
+// Unwrap returns wrapped error
+//
+// This function is auto-generated.
+//
+func (e *{{ $.Service }}Error) Unwrap() error {
+	return e.wrap
+}
+
+func (e *{{ $.Service }}Error) LoggableAction() *actionlog.Action {
+	return &actionlog.Action{
+		Timestamp:   e.timestamp,
+		Resource:    e.resource,
+		Action:      e.action,
+		Severity:    e.severity,
+		Description: e.String(),
+		Error:       e.Error(),
+		Meta:        e.props.serialize(),
+	}
+}
+
+{{ if $.SupportHttpErrors }}
+func (e *{{ $.Service }}Error) HttpResponse(w http.ResponseWriter) {
+    var code = e.httpStatusCode
+    if code == 0 {
+        code = http.StatusInternalServerError
+    }
+
+	http.Error(w, e.message, code)
+}
+{{ end }}
+
+{{ end }}
+
+{{ if $.Actions }}
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+// Action constructors
+
+{{ range $a := $.Actions }}
+// {{ camelCase "" $.Service "Action" $a.Action  }} returns "{{ $.Resource }}.{{ $a.Action  }}" error
+//
+// This function is auto-generated.
+//
+func {{ camelCase "" $.Service "Action" $a.Action  }}(props ... *{{ $.Service }}ActionProps) *{{ $.Service }}Action {
+	a := &{{ $.Service }}Action{
+		timestamp:    time.Now(),
+		resource:     "{{ $.Resource }}",
+		action:       "{{ $a.Action }}",
+		log:   "{{ $a.Log }}",
+		severity:     {{ $a.SeverityConstName }},
+	}
+
+	if len(props) > 0 {
+	    a.props = props[0]
+	}
+
+	return a
+}
+{{ end }}
+{{ end }}
+
+{{ if $.Errors }}
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+// Error constructors
+
+{{ range $e := $.Errors }}
+{{- if $e.Safe }}
+// {{ camelCase "" $.Service "Err" $e.Error }} returns "{{ $.Resource }}.{{ $e.Safe  }}" audit event as {{ $e.SeverityConstName }}
+{{- else }}
+// {{ camelCase "" $.Service "Err" $e.Error }} returns "{{ $.Resource }}.{{ $e.Error  }}" audit event as {{ $e.SeverityConstName }}
+{{- end }}
+//
+{{- if $e.Safe }}
+// Note: This error will be wrapped with safe ({{ $e.Safe }}) error!
+{{- end }}
+//
+// This function is auto-generated.
+//
+func {{ camelCase "" $.Service "Err" $e.Error }}(props ... *{{ $.Service }}ActionProps) *{{ $.Service }}Error {
+	var e = &{{ $.Service }}Error{
+		timestamp: time.Now(),
+		resource:  "{{ $.Resource }}",
+		error:     "{{ $e.Error }}",
+		action:    "error",
+		message:   "{{ $e.Message }}",
+		log:       "{{ $e.Log }}",
+		severity:  {{ $e.SeverityConstName }},
+		props:     func() *{{ $.Service }}ActionProps { if len(props) > 0 { return props[0] }; return nil}(),
+
+    {{ if $e.HttpStatus }}
+        httpStatusCode: http.{{ $e.HttpStatus }},
+    {{ end }}
+	}
+
+	if len(props) > 0 {
+	    e.props = props[0]
+	}
+
+	{{ if $e.Safe }}
+	// Wrap with safe error
+	return {{ camelCase "" $.Service "Err" $e.Safe }}().Wrap(e)
+	{{ else }}
+	return e
+	{{ end }}
+}
+{{ end }}
+{{ end }}
+
+// *********************************************************************************************************************
+// *********************************************************************************************************************
+
+// recordAction is a service helper function wraps function that can return error
+//
+// context is used to enrich audit log entry with current user info, request ID, IP address...
+// props are collected action/error properties
+// action (optional) fn will be used to construct {{ $.Service }}Action struct from given props (and error)
+// err is any error that occurred while action was happening
+//
+// Action has success and fail (error) state:
+//  - when recorded without an error (4th param), action is recorded as successful.
+//  - when an additional error is given (4th param), action is used to wrap
+//    the additional error
+//
+// This function is auto-generated.
+//
+func (svc {{ $.Service }}) recordAction(ctx context.Context, props *{{ $.Service }}ActionProps, action func(... *{{ $.Service }}ActionProps) *{{ $.Service }}Action, err error) error {
+	var (
+		ok bool
+
+		// Return error
+		retError *{{ $.Service }}Error
+
+		// Recorder error
+		recError *{{ $.Service }}Error
+	)
+
+	if err != nil {
+		if retError, ok = err.(*{{ $.Service }}Error); !ok {
+			// got non-{{ $.Service }} error, wrap it with {{ camelCase "" $.Service "err" "generic" }}
+			retError = {{ camelCase "" $.Service "err" "generic" }}(props).Wrap(err)
+
+			if action != nil {
+			    // copy action to returning and recording error
+			    retError.action = action().action
+			}
+
+			// we'll use {{ camelCase "" $.Service "err" "generic" }} for recording too
+			// because it can hold more info
+			recError = retError
+		} else if retError != nil {
+			if action != nil {
+                // copy action to returning and recording error
+                retError.action = action().action
+            }
+			// start with copy of return error for recording
+			// this will be updated with tha root cause as we try and
+			// unwrap the error
+			recError = retError
+
+			// find the original recError for this error
+			// for the purpose of logging
+			var unwrappedError error = retError
+			for {
+				if unwrappedError = errors.Unwrap(unwrappedError); unwrappedError == nil {
+					// nothing wrapped
+					break
+				}
+
+				// update recError ONLY of wrapped error is of type {{ $.Service }}Error
+				if unwrappedSinkError, ok := unwrappedError.(*{{ $.Service }}Error); ok {
+					recError = unwrappedSinkError
+				}
+			}
+
+			if retError.props == nil {
+				// set props on returning error if empty
+				retError.props = props
+			}
+
+			if recError.props == nil {
+				// set props on recording error if empty
+				recError.props = props
+			}
+		}
+	}
+
+	if svc.actionlog != nil {
+		if retError != nil {
+			// failed action, log error
+			svc.actionlog.Record(ctx, recError)
+		} else if action != nil {
+			// successful
+			svc.actionlog.Record(ctx, action(props))
+		}
+	}
+
+	if err == nil {
+		// retError not an interface and that WILL (!!) cause issues
+		// with nil check (== nil) when it is not explicitly returned
+		return nil
+	}
+
+	return retError
+}

codegen/v2/actionlog/main.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/cortezaproject/corteza-server/codegen/v2/internal"
+	"github.com/cortezaproject/corteza-server/pkg/cli"
+	"github.com/cortezaproject/corteza-server/pkg/handle"
+
+	"gopkg.in/yaml.v2"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/template"
+)
+
+const (
+	eventsTemplateFile = "codegen/v2/actionlog/*.go.tpl"
+)
+
+type (
+	// List of event/log properties that can/will be captured
+	// and injected into log or message string
+	propsDef struct {
+		Name    string
+		Type    string
+		Fields  []string
+		Builtin bool
+	}
+
+	actionDef struct {
+		// Action name
+		Action string `yaml:"action"`
+
+		// String to log when action is successful
+		Log string `yaml:"log"`
+
+		// String to log when error was yield
+		//ErrorLog string `yaml:"errorLog"`
+
+		// Action severity
+		Severity string `yaml:"severity"`
+	}
+
+	// Event definition
+	errorDef struct {
+		// Error key
+		// message can contain {variables} from meta data
+		Error string `yaml:"error"`
+
+		// Error key
+		// message can contain {variables} from meta data
+		Message string `yaml:"message"`
+
+		// Formatted and readable audit log message
+		// message can contain {variables} from meta data
+		Log string `yaml:"log"`
+
+		// Reference to "safe" error
+		// safe error should hide any information that might cause
+		// personal data leakage or expose system internals
+		Safe string `yaml:"safe"`
+
+		// Error severity
+		Severity string `yaml:"severity"`
+
+		// HTTP Status code for this error
+		HttpStatus string `yaml:"httpStatus"`
+	}
+)
+
+const (
+	// list of actinos and errors
+	defSuffix = "_actions.yaml"
+)
+
+var (
+	// Cut off this binary
+	defs = os.Args[1:]
+
+	overwrite bool
+	preview   bool
+
+	tpl *template.Template
+
+	placeholderMatcher = regexp.MustCompile(`{(.+?)}`)
+)
+
+func main() {
+	tpl = template.New("").Funcs(map[string]interface{}{
+		"camelCase": internal.CamelCase,
+	})
+
+	tpl = template.Must(tpl.ParseGlob(eventsTemplateFile))
+
+	flag.BoolVar(&overwrite, "overwrite", false, "Overwrite all files")
+	flag.BoolVar(&preview, "preview", false, "Output to stdout instead of outputPath")
+	flag.Parse()
+
+	for _, path := range defs {
+		defs, err := filepath.Glob(path + "/*" + defSuffix)
+		if err != nil {
+			cli.HandleError(err)
+		}
+
+		for _, def := range defs {
+			base := filepath.Base(def)
+			procDef(def, filepath.Join(filepath.Dir(def), base[0:len(base)-len(defSuffix)]+"_actions.gen.go"))
+		}
+	}
+}
+
+func procDef(path, output string) {
+	println(path, output)
+	var (
+		decoder *yaml.Decoder
+		tplData = struct {
+			Command string
+			YAML    string
+
+			Package string
+
+			// List of imports
+			// Used only by generated file and not pre-generated-user-file
+			Import []string `yaml:"import"`
+
+			Service  string `yaml:"service"`
+			Resource string `yaml:"resource"`
+
+			// Default severity for actions
+			DefaultActionSeverity string `yaml:"defaultActionSeverity"`
+
+			// Default severity for errors
+			DefaultErrorSeverity string `yaml:"defaultErrorSeverity"`
+
+			// If at least one of the errors has HTTP status defined,
+			// add support for http errors
+			SupportHttpErrors bool
+
+			Props   []*propsDef
+			Actions []*actionDef
+			Errors  []*errorDef
+		}{
+			Package: "service",
+			YAML:    path,
+
+			DefaultActionSeverity: "info",
+			DefaultErrorSeverity:  "error",
+		}
+	)
+
+	if f, err := os.Open(path); err != nil {
+		cli.HandleError(err)
+	} else {
+		decoder = yaml.NewDecoder(f)
+	}
+
+	cli.HandleError(decoder.Decode(&tplData))
+
+	// Prepend generic error
+	tplData.Errors = append([]*errorDef{{
+		Error:    "generic",
+		Message:  "failed to complete request due to internal error",
+		Log:      "{err}",
+		Severity: "error",
+	}}, tplData.Errors...)
+
+	for i := range tplData.Import {
+		// Handle list of imports, adds quotes around each import
+		//
+		// If import string contains a space, assume import alias and
+		// quotes only the 2nd part
+		if strings.Contains(tplData.Import[i], " ") {
+			p := strings.SplitN(tplData.Import[i], " ", 2)
+			tplData.Import[i] = fmt.Sprintf(`%s "%s"`, p[0], p[1])
+		} else {
+			tplData.Import[i] = fmt.Sprintf(`"%s"`, tplData.Import[i])
+		}
+	}
+
+	// index known meta fields and sanitize types (no type => string type)
+	knownProps := map[string]bool{
+		"err": true,
+	}
+
+	for _, m := range tplData.Props {
+		if m.Type == "" {
+			m.Type = "string"
+		}
+
+		// very optimistic check if referenced type is builtin or not
+		m.Builtin = !strings.Contains(m.Type, ".")
+
+		knownProps[m.Name] = true
+
+		for _, f := range m.Fields {
+			knownProps[fmt.Sprintf("%s.%s", m.Name, f)] = true
+
+		}
+	}
+
+	for _, a := range tplData.Actions {
+		if a.Severity == "" {
+			a.Severity = tplData.DefaultActionSeverity
+		}
+	}
+
+	for _, e := range tplData.Errors {
+		if e.Severity == "" {
+			e.Severity = tplData.DefaultErrorSeverity
+		}
+
+		if e.HttpStatus != "" {
+			tplData.SupportHttpErrors = true
+		}
+	}
+
+	checkHandle := func(s string) {
+		if !handle.IsValid(s) {
+			cli.HandleError(fmt.Errorf(
+				"%s: %s handle empty", path))
+
+		}
+
+		if !handle.IsValid(s) {
+			cli.HandleError(fmt.Errorf(
+				"%s: invalid handle format: %q", path, s))
+
+		}
+	}
+
+	checkPlaceholders := func(def string, kind, s string) {
+		for _, match := range placeholderMatcher.FindAllStringSubmatch(s, 1) {
+			placeholder := match[1]
+			if !knownProps[placeholder] {
+				cli.HandleError(fmt.Errorf(
+					"%s: unknown placeholder %q used in %s for %s", path, placeholder, def, kind))
+			}
+		}
+	}
+
+	for _, a := range tplData.Actions {
+		checkHandle(a.Action)
+		if a.Log == "" {
+			// If no log is defined, use action handle
+			a.Log = a.Action
+		}
+
+		checkPlaceholders(a.Action, "log", a.Log)
+	}
+
+	for _, e := range tplData.Errors {
+		checkHandle(e.Error)
+
+		if e.Message == "" {
+			// If no error message is defined, use error handle
+			e.Message = e.Error
+		}
+
+		if e.Log == "" {
+			// If no error log is defined, use error message
+			e.Log = e.Message
+		}
+
+		checkPlaceholders(e.Error, "message", e.Message)
+		checkPlaceholders(e.Error, "log", e.Log)
+	}
+
+	internal.WriteTo(tpl, tplData, "actions.gen.go.tpl", output)
+}
+
+func (a actionDef) SeverityConstName() string {
+	return severityConstName(a.Severity)
+}
+
+func (e errorDef) SeverityConstName() string {
+	return severityConstName(e.Severity)
+}
+
+func severityConstName(s string) string {
+	switch strings.ToLower(s) {
+	case "emergency":
+		return "actionlog.Emergency"
+	case "alert":
+		return "actionlog.Alert"
+	case "crit", "critical":
+		return "actionlog.Critical"
+	case "warn", "warning":
+		return "actionlog.Warning"
+	case "notice":
+		return "actionlog.Notice"
+	case "info", "informational":
+		return "actionlog.Info"
+	case "debug":
+		return "actionlog.Debug"
+	default:
+		return "actionlog.Error"
+	}
+}

codegen/v2/events.go → codegen/v2/events/main.go

 package main
 
-// *Set type generator for functions like Walk(), Filter(), FindByID() & IDs()
-
 import (
-	"bytes"
 	"flag"
 	"fmt"
-	"go/format"
-	"io"
+	"github.com/cortezaproject/corteza-server/codegen/v2/internal"
+	"github.com/cortezaproject/corteza-server/pkg/cli"
+	"gopkg.in/yaml.v2"
 	"os"
 	"strings"
 	"text/template"
-
-	"gopkg.in/yaml.v2"
-
-	"github.com/cortezaproject/corteza-server/pkg/cli"
 )
 
 const (
-	eventsTemplateFile = "codegen/v2/*.go.tpl"
+	templateFile = "codegen/v2/events/*.go.tpl"
 )
 
 type (
@@ -67,11 +61,11 @@ type (
 func main() {
 
 	tpl := template.New("").Funcs(map[string]interface{}{
-		"camelCase":  camelCase,
+		"camelCase":  internal.CamelCase,
 		"makeEvents": makeEvents,
 	})
 
-	tpl = template.Must(tpl.ParseGlob(eventsTemplateFile))
+	tpl = template.Must(tpl.ParseGlob(templateFile))
 
 	var (
 		definitionsPathStr string
@@ -113,7 +107,7 @@ func main() {
 		decoder = yaml.NewDecoder(f)
 	}
 
-	tplData.Command = "go run codegen/v2/events.go --service " + serviceStr
+	tplData.Command = "go run ./codegen/v2/events --service " + serviceStr
 	tplData.YAML = definitionsPathStr + yamlDefFileName
 	tplData.Package = "event"
 
@@ -136,7 +130,7 @@ func main() {
 			// check if resource name is shorter  and has invalid prefix
 			cli.HandleError(fmt.Errorf("invalid resource prefix: %q", resName))
 		} else {
-			tplData.ResourceIdent = camelCase(strings.Split(resName[l+1:], ":")...)
+			tplData.ResourceIdent = internal.CamelCase(strings.Split(resName[l+1:], ":")...)
 
 			fname = resName[l+1:]
 			fname = strings.ReplaceAll(fname, ":", "_")
@@ -187,10 +181,10 @@ func main() {
 
 		_, err := os.Stat(usrOutput)
 		if overwrite || os.IsNotExist(err) {
-			writeTo(tpl, tplData, "events.go.tpl", usrOutput)
+			internal.WriteTo(tpl, tplData, "events.go.tpl", usrOutput)
 		}
 
-		writeTo(tpl, tplData, "events.gen.go.tpl", genOutput)
+		internal.WriteTo(tpl, tplData, "events.gen.go.tpl", genOutput)
 	}
 }
 
@@ -215,18 +209,6 @@ func makeEvents(def eventDef) []string {
 	)
 }
 
-func camelCase(pp ...string) (out string) {
-	for i, p := range pp {
-		if i > 0 && len(p) > 1 {
-			p = strings.ToUpper(p[:1]) + p[1:]
-		}
-
-		out = out + p
-	}
-
-	return out
-}
-
 func makeEventGroup(pfix string, ee []string) (out []string) {
 	for _, e := range ee {
 		out = append(out, pfix+strings.ToUpper(e[:1])+e[1:])
@@ -234,33 +216,3 @@ func makeEventGroup(pfix string, ee []string) (out []string) {
 
 	return
 }
-
-func writeTo(tpl *template.Template, payload interface{}, name, dst string) {
-	var output io.WriteCloser
-	buf := bytes.Buffer{}
-
-	if err := tpl.ExecuteTemplate(&buf, name, payload); err != nil {
-		cli.HandleError(err)
-	} else {
-		fmtsrc, err := format.Source(buf.Bytes())
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "fmt warn: %v", err)
-			fmtsrc = buf.Bytes()
-		}
-
-		if dst == "" || dst == "-" {
-			output = os.Stdout
-		} else {
-			// cli.HandleError(os.Remove(dst))
-			if output, err = os.Create(dst); err != nil {
-				cli.HandleError(err)
-			}
-
-			defer output.Close()
-		}
-
-		if _, err := output.Write(fmtsrc); err != nil {
-			cli.HandleError(err)
-		}
-	}
-}

codegen/v2/internal/strings.go

+package internal
+
+import "strings"
+
+func CamelCase(pp ...string) (out string) {
+	for i, p := range pp {
+		if i > 0 && len(p) > 1 {
+			p = strings.ToUpper(p[:1]) + p[1:]
+		}
+
+		out = out + p
+	}
+
+	return out
+}

codegen/v2/internal/templating.go

+package internal
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/cortezaproject/corteza-server/pkg/cli"
+	"go/format"
+	"io"
+	"os"
+	"text/template"
+)
+
+func WriteTo(tpl *template.Template, payload interface{}, name, dst string) {
+	var output io.WriteCloser
+	buf := bytes.Buffer{}
+
+	if err := tpl.ExecuteTemplate(&buf, name, payload); err != nil {
+		cli.HandleError(err)
+	} else {
+		fmtsrc, err := format.Source(buf.Bytes())
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "fmt warn: %v", err)
+			fmtsrc = buf.Bytes()
+		}
+
+		if dst == "" || dst == "-" {
+			output = os.Stdout
+		} else {
+			// cli.HandleError(os.Remove(dst))
+			if output, err = os.Create(dst); err != nil {
+				cli.HandleError(err)
+			}
+
+			defer output.Close()
+		}
+
+		if _, err := output.Write(fmtsrc); err != nil {
+			cli.HandleError(err)
+		}
+	}
+}

docs/system/README.md

+# Action log
+
+| Method | Endpoint | Purpose |
+| ------ | -------- | ------- |
+| `GET` | `/actionlog/` | Action log events |
+
+## Action log events
+
+#### Method
+
+| URI | Protocol | Method | Authentication |
+| --- | -------- | ------ | -------------- |
+| `/actionlog/` | HTTP/S | GET |
+Warning: implode(): Invalid arguments passed in /private/tmp/Users/darh/Work.crust/corteza-server/codegen/templates/README.tpl on line 32
+ |
+
+#### Request parameters
+
+| Parameter | Type | Method | Description | Default | Required? |
+| --------- | ---- | ------ | ----------- | ------- | --------- |
+| from | *time.Time | GET | From | N/A | NO |
+| to | *time.Time | GET | To | N/A | NO |
+| resource | string | GET | Resource | N/A | NO |
+| action | string | GET | Action | N/A | NO |
+| actorID | []string | GET | Filter by one or more actors | N/A | NO |
+| limit | uint | GET | Limit | N/A | NO |
+| offset | uint | GET | Offset | N/A | NO |
+| page | uint | GET | Page number (1-based) | N/A | NO |
+| perPage | uint | GET | Returned items per page (default 50) | N/A | NO |
+
+---
+
+
+
+
 # Applications
 
 | Method | Endpoint | Purpose |

pkg/actionlog/context.go

+package actionlog
+
+import (
+	"context"
+)
+
+// Key to use when setting the request ID.
+type ctxKey int
+
+const (
+	RequestOrigin_APP_Init            = "app/init"
+	RequestOrigin_APP_Serve           = "app/serve"
+	RequestOrigin_APP_Upgrade         = "app/upgrade"
+	RequestOrigin_APP_Activate        = "app/activate"
+	RequestOrigin_APP_Provision       = "app/provision"
+	RequestOrigin_APP_Run             = "app/run"
+	RequestOrigin_HTTPServer_API_REST = "app/http-server/api/rest"
+	RequestOrigin_HTTPServer_API_GRPC = "app/http-server/api/grpc"
+	RequestOrigin_CLI                 = "app/cli"
+)
+
+// RequestOriginKey is the key that holds th unique request ID in a request context.
+const requestOriginKey ctxKey = 0
+
+// RequestOriginToContext stores request origin to context
+func RequestOriginToContext(ctx context.Context, origin string) context.Context {
+	return context.WithValue(ctx, requestOriginKey, origin)
+}
+
+// RequestOriginFromContext returns remote IP address from context
+func RequestOriginFromContext(ctx context.Context) string {
+	v := ctx.Value(requestOriginKey)
+	if str, ok := v.(string); ok {
+		return str
+	}
+
+	return ""
+}

pkg/actionlog/repository/mysql.go

+package repository
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/Masterminds/squirrel"
+	"github.com/cortezaproject/corteza-server/pkg/actionlog"
+	"github.com/cortezaproject/corteza-server/pkg/rh"
+	"github.com/titpetric/factory"
+	"time"
+)
+
+type (
+	// Basic mysql storage backend for audit log events
+	//
+	// this does not follow the usual (one) repository pattern
+	// but tries to move towards multi-flavoured repository support
+	mysql struct {
+		dbh *factory.DB
+		tbl string
+	}
+
+	event struct {
+		Timestamp     time.Time       `db:"ts"`
+		RequestOrigin string          `db:"request_origin"`
+		RequestID     string          `db:"request_id"`
+		ActorIPAddr   string          `db:"actor_ip_addr"`
+		ActorID       uint64          `db:"actor_id"`
+		Resource      string          `db:"resource"`
+		Action        string          `db:"action"`
+		Error         string          `db:"error"`
+		Severity      int             `db:"severity"`
+		Description   string          `db:"description"`
+		Meta          json.RawMessage `db:"meta"`
+	}
+)
+
+func Mysql(db *factory.DB, tbl string) *mysql {
+	return &mysql{
+		// connection
+		dbh: db,
+
+		// table to store the data
+		tbl: tbl,
+	}
+}
+
+func (r *mysql) db() *factory.DB {
+	return r.dbh
+}
+
+func (r mysql) columns() []string {
+	return []string{
+		"ts",
+		"request_origin",
+		"request_id",
+		"actor_ip_addr",
+		"actor_id",
+		"resource",
+		"action",
+		"error",
+		"severity",
+		"description",
+		"meta",
+	}
+}
+
+func (r mysql) query() squirrel.SelectBuilder {
+	return squirrel.
+		Select(r.columns()...).
+		From(r.tbl)
+}
+
+func (r *mysql) Find(ctx context.Context, flt actionlog.Filter) (set actionlog.ActionSet, f actionlog.Filter, err error) {
+	f = flt
+
+	query := r.query()
+
+	if f.From != nil {
+		query = query.Where(squirrel.GtOrEq{"ts": f.From})
+	}
+
+	if f.To != nil {
+		query = query.Where(squirrel.LtOrEq{"ts": f.To})
+	}
+
+	if len(f.ActorID) > 0 {
+		query = query.Where(squirrel.Eq{"actor_id": f.ActorID})
+	}
+
+	if f.Resource != "" {
+		query = query.Where(squirrel.Eq{"resource": f.Resource})
+	}
+
+	if f.Action != "" {
+		query = query.Where(squirrel.Eq{"action": f.Action})
+	}
+
+	// @todo implement filtering with query (via pkg/ql)
+
+	query = query.OrderBy("ts DESC")
+
+	results := make([]*event, 0)
+	if err = rh.FetchPaged(r.db(), query, f.PageFilter, &results); err != nil {
+		return nil, f, err
+	}
+
+	set = make(actionlog.ActionSet, len(results))
+	for i, r := range results {
+		set[i] = &actionlog.Action{
+			Timestamp:     r.Timestamp,
+			RequestOrigin: r.RequestOrigin,
+			RequestID:     r.RequestID,
+			ActorIPAddr:   r.ActorIPAddr,
+			ActorID:       r.ActorID,
+			Resource:      r.Resource,
+			Action:        r.Action,
+			Error:         r.Error,
+			Severity:      actionlog.Severity(r.Severity),
+			Description:   r.Description,
+		}
+
+		// ignore all unmarshaling issues
+		_ = json.Unmarshal(r.Meta, &set[i].Meta)
+	}
+
+	return set, f, nil
+}
+
+// Record stores audit event
+func (r *mysql) Record(ctx context.Context, e *actionlog.Action) error {
+	m, err := json.Marshal(e.Meta)
+	if err != nil {
+		return fmt.Errorf("could not format auditlog event: %w", err)
+	}
+
+	return r.dbh.With(ctx).InsertIgnore(r.tbl, event{
+		Timestamp:     e.Timestamp,
+		RequestOrigin: e.RequestOrigin,
+		RequestID:     e.RequestID,
+		ActorIPAddr:   e.ActorIPAddr,
+		ActorID:       e.ActorID,
+		Resource:      e.Resource,
+		Action:        e.Action,
+		Error:         e.Error,
+		Severity:      int(e.Severity),
+		Description:   e.Description,
+		Meta:          m,
+	})
+}

pkg/actionlog/service.go

+package actionlog
+
+import (
+	"context"
+	"strings"
+
+	"github.com/go-chi/chi/middleware"
+
+	"github.com/cortezaproject/corteza-server/pkg/api"
+	"github.com/cortezaproject/corteza-server/pkg/auth"
+
+	"go.uber.org/zap"
+)
+
+type (
+	service struct {
+		// where the audit log records are kept
+		repo recordKeeper
+
+		// Also write audit events here
+		tee *zap.Logger
+
+		// logger for repository errors
+		logger *zap.Logger
+	}
+
+	loggable interface {
+		LoggableAction() *Action
+	}
+
+	Recorder interface {
+		Record(context.Context, loggable)
+		Find(context.Context, Filter) (ActionSet, Filter, error)
+	}
+
+	recordKeeper interface {
+		Record(context.Context, *Action) error
+		Find(context.Context, Filter) (ActionSet, Filter, error)
+	}
+)
+
+// NewService initializes auditlog service
+//
+func NewService(r recordKeeper, logger, tee *zap.Logger) (svc *service) {
+	if tee == nil {
+		tee = zap.NewNop()
+	}
+
+	svc = &service{
+		tee:    tee,
+		logger: logger,
+		repo:   r,
+	}
+
+	return
+}
+
+func (svc service) Record(ctx context.Context, l loggable) {
+	if l == nil {
+		// nothing to record
+		return
+	}
+
+	a := enrich(ctx, l.LoggableAction())
+
+	var (
+		log = svc.logger
+	)
+
+	zlf := []zap.Field{
+		zap.Time("timestamp", a.Timestamp),
+		zap.String("requestOrigin", a.RequestOrigin),
+		zap.String("requestID", a.RequestID),
+		zap.String("actorIPAddr", a.ActorIPAddr),
+		zap.Uint64("actorID", a.ActorID),
+		zap.String("resource", a.Resource),
+		zap.String("action", a.Action),
+		zap.Uint8("severity", uint8(a.Severity)),
+		zap.String("error", a.Error),
+		zap.String("description", a.Description),
+		zap.Any("meta", a.Meta),
+	}
+
+	for k, v := range a.Meta {
+		zlf = append(zlf, zap.Any("meta."+k, v))
+	}
+
+	log.Debug(a.Description, zlf...)
+
+	if err := svc.repo.Record(ctx, a); err != nil {
+		log.With(zap.Error(err)).Error("could not record audit event")
+	}
+}
+
+func (svc service) Find(ctx context.Context, flt Filter) (ActionSet, Filter, error) {
+	return svc.repo.Find(ctx, flt)
+}
+
+// Enriches action with additional info (ip, actor id, request id...)
+func enrich(ctx context.Context, a *Action) *Action {
+	a.RequestOrigin = RequestOriginFromContext(ctx)
+
+	// Relies on chi's middleware to get to the request ID
+	// This does not hurt us for now.
+	a.RequestID = middleware.GetReqID(ctx)
+
+	// uses pkg/auth to extract stored identity from context
+	a.ActorID = auth.GetIdentityFromContext(ctx).Identity()
+
+	// IP from the request,
+	// we're splitting by space & colon to remove any additional (proxy) IPs
+	// and ports from the string
+	if tmp := strings.SplitN(api.RemoteAddrFromContext(ctx), " ", 2); len(tmp) > 0 {
+		// split by : (ip:port)
+		if tmp = strings.SplitN(tmp[0], ":", 2); len(tmp) > 0 {
+			const maxLen = 16
+			ipAddr := tmp[0]
+			if len(ipAddr) > maxLen {
+				ipAddr = ipAddr[:maxLen-1]
+			}
+
+			a.ActorIPAddr = ipAddr
+		}
+	}
+
+	return a
+}

pkg/actionlog/types.gen.go

+package actionlog
+
+// 	Hello! This file is auto-generated.
+
+type (
+
+	// ActionSet slice of Action
+	//
+	// This type is auto-generated.
+	ActionSet []*Action
+)
+
+// Walk iterates through every slice item and calls w(Action) err
+//
+// This function is auto-generated.
+func (set ActionSet) Walk(w func(*Action) error) (err error) {
+	for i := range set {
+		if err = w(set[i]); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// Filter iterates through every slice item, calls f(Action) (bool, err) and return filtered slice
+//
+// This function is auto-generated.
+func (set ActionSet) Filter(f func(*Action) (bool, error)) (out ActionSet, err error) {
+	var ok bool
+	out = ActionSet{}
+	for i := range set {
+		if ok, err = f(set[i]); err != nil {
+			return
+		} else if ok {
+			out = append(out, set[i])
+		}
+	}
+
+	return
+}

pkg/actionlog/types.gen_test.go

+package actionlog
+
+import (
+	"testing"
+
+	"errors"
+
+	"github.com/stretchr/testify/require"
+)
+
+// 	Hello! This file is auto-generated.
+
+func TestActionSetWalk(t *testing.T) {
+	var (
+		value = make(ActionSet, 3)
+		req   = require.New(t)
+	)
+
+	// check walk with no errors
+	{
+		err := value.Walk(func(*Action) error {
+			return nil
+		})
+		req.NoError(err)
+	}
+
+	// check walk with error
+	req.Error(value.Walk(func(*Action) error { return errors.New("walk error") }))
+
+}
+
+func TestActionSetFilter(t *testing.T) {
+	var (
+		value = make(ActionSet, 3)
+		req   = require.New(t)
+	)
+
+	// filter nothing
+	{
+		set, err := value.Filter(func(*Action) (bool, error) {
+			return true, nil
+		})
+		req.NoError(err)
+		req.Equal(len(set), len(value))
+	}
+
+	// filter one item
+	{
+		found := false
+		set, err := value.Filter(func(*Action) (bool, error) {
+			if !found {
+				found = true
+				return found, nil
+			}
+			return false, nil
+		})
+		req.NoError(err)
+		req.Len(set, 1)
+	}
+
+	// filter error
+	{
+		_, err := value.Filter(func(*Action) (bool, error) {
+			return false, errors.New("filter error")
+		})
+		req.Error(err)
+	}
+}

pkg/actionlog/types.go

+package actionlog
+
+import (
+	"github.com/cortezaproject/corteza-server/pkg/rh"
+	"time"
+)
+
+type (
+	// Any additional data
+	// that can be packed with the raised audit event
+	Meta map[string]string
+
+	// Severity determinants event severity level
+	Severity uint8
+
+	// Standardized data structure for audit log events
+	Action struct {
+		// Timestamp of the raised event
+		Timestamp time.Time `json:"timestamp"`
+
+		// Origin of the action (rest-api, cli, grpc, system)
+		RequestOrigin string `json:"requestOrigin"`
+
+		// Request ID
+		RequestID string `json:"requestID"`
+
+		// This can contain a series of IP addresses (when proxied)
+		// https://en.wikipedia.org/wiki/X-Forwarded-For#Format
+		ActorIPAddr string `json:"actorIPAddr"`
+
+		// ID of the user (if not anonymous)
+		ActorID uint64 `json:"actorID,string"`
+
+		// Resource
+		Resource string `json:"resource"`
+
+		// Type of action
+		Action string `json:"action"`
+
+		// Type of error
+		Error string `json:"error"`
+
+		// Action severity
+		Severity Severity `json:"severity"`
+
+		// Description of the event
+		Description string `json:"description"`
+
+		// Meta data, resource specific values
+		Meta Meta `json:"meta"`
+	}
+
+	Filter struct {
+		From     *time.Time `json:"from"`
+		To       *time.Time `json:"to"`
+		ActorID  []uint64   `json:"actorID"`
+		Resource string     `json:"resource"`
+		Action   string     `json:"action"`
+
+		// @todo pending implementation
+		// Query   string     `json:"query"`
+
+		// Standard paging fields & helpers
+		rh.PageFilter
+	}
+)
+
+const (
+	// Not using log/syslog LOG_* constants as they are only
+	// available outside windows env.
+	Emergency Severity = iota
+	Alert
+	Critical
+	Error
+	Warning
+	Notice
+	Info
+	Debug
+)
+
+func (a *Action) LoggableAction() *Action { return a }

pkg/api/ipaddr.go

+package api
+
+import (
+	"context"
+	"net/http"
+)
+
+// Key to use when setting the request ID.
+type ctxKeyRemoteAddr int
+
+// RemoteAddrKey is the key that holds th unique request ID in a request context.
+const remoteAddrKey ctxKeyRemoteAddr = 0
+
+// Packs remote address to context
+func remoteAddrToContext(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		next.ServeHTTP(w, req.WithContext(context.WithValue(req.Context(), remoteAddrKey, req.RemoteAddr)))
+	})
+}
+
+// RemoteAddrFromContext returns remote IP address from context
+func RemoteAddrFromContext(ctx context.Context) string {
+	v := ctx.Value(remoteAddrKey)
+	if str, ok := v.(string); ok {
+		return str
+	}
+
+	return ""
+}

pkg/api/logger.go

@@ -14,7 +14,7 @@ import (
 // contextLogger middleware binds logger to request's context.
 //
 // This allows us to use logger from context (with requestID)
-// inside our (generated) handers and controllers
+// inside our (generated) handlers and controllers
 func contextLogger(log *zap.Logger) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

pkg/api/middleware.go

@@ -17,6 +17,7 @@ func BaseMiddleware(log *zap.Logger) []func(http.Handler) http.Handler {
 	return []func(http.Handler) http.Handler{
 		handleCORS,
 		middleware.RealIP,
+		remoteAddrToContext,
 		middleware.RequestID,
 		contextLogger(log),
 	}