Sensu event doesnt get silenced

(Arunn Srinivasan) #1

Hi I am new to stackstorm and sensu.
I have installed sensu and stackstorm on two different vm’s, i have script on sensu which generates checkto sensu, these are reaching stackstorm as trigger, i am using the sensu rules to silence some events

example below
I see the rule is matched and executed as success, but on the sensu side the check is not getting sileneced

{
  "uid": "rule:sensu:Sample_HA_1",
  "tags": [],
  "type": {
    "ref": "standard",
    "parameters": {}
  },
  "enabled": true,
  "trigger": {
    "type": "sensu.event_handler",
    "ref": "sensu.event_handler",
    "description": "Trigger type for sensu event handler.",
    "parameters": {}
  },
  "metadata_file": "",
  "context": {
    "user": "st2admin"
  },
  "criteria": {
    "trigger.check.output": {
      "pattern": "craigslist",
      "type": "contains"
    }
  },
  "action": {
    "ref": "sensu.silence",
    "description": "Silence a Sensu client or check",
    "parameters": {
      "log_level": "DEBUG",
      "client": "abc"
    }
  },
  "pack": "sensu",
  "ref": "sensu.Sample_HA_1",
  "id": "5cccaa896cb8de52d1d6b732",
  "name": "Sample_HA_1"
}

output

{
  "status": "succeeded",
  "start_timestamp": "2019-05-04T01:06:01.485339Z",
  "log": [
    {
      "status": "requested",
      "timestamp": "2019-05-04T01:06:01.000000Z"
    },
    {
      "status": "scheduled",
      "timestamp": "2019-05-04T01:06:01.000000Z"
    },
    {
      "status": "running",
      "timestamp": "2019-05-04T01:06:01.000000Z"
    },
    {
      "status": "succeeded",
      "timestamp": "2019-05-04T01:06:01.000000Z"
    }
  ],
  "parameters": {
    "message": "arun",
    "client": "cyr.qa.craigslist-456015.EC2.SFW.SFW_110406_us-east-2_craigslist-456015",
    "log_level": "DEBUG",
    "check": "ha_status"
  },
  "runner": {
    "runner_module": "python_runner",
    "uid": "runner_type:python-script",
    "runner_package": "python_runner",
    "description": "A runner for launching python actions.",
    "enabled": true,
    "output_key": "result",
    "output_schema": {
      "result": {
        "anyOf": [
          {
            "type": "object"
          },
          {
            "type": "string"
          },
          {
            "type": "integer"
          },
          {
            "type": "number"
          },
          {
            "type": "boolean"
          },
          {
            "type": "array"
          },
          {
            "type": "null"
          }
        ]
      },
      "exit_code": {
        "required": true,
        "type": "integer"
      },
      "stderr": {
        "required": true,
        "type": "string"
      },
      "stdout": {
        "required": true,
        "type": "string"
      }
    },
    "runner_parameters": {
      "debug": {
        "default": false,
        "required": false,
        "type": "boolean",
        "description": "Enable runner debug mode.",
        "_name": "debug"
      },
      "content_version": {
        "required": false,
        "type": "string",
        "description": "Git revision of the pack content to use for this action execution (git commit sha / tag / branch). Only applies to packs which are git repositories.",
        "_name": "content_version"
      },
      "log_level": {
        "default": "DEBUG",
        "enum": [
          "AUDIT",
          "CRITICAL",
          "ERROR",
          "WARNING",
          "INFO",
          "DEBUG"
        ],
        "type": "string",
        "description": "Default log level for Python runner actions.",
        "_name": "log_level"
      },
      "timeout": {
        "default": 600,
        "type": "integer",
        "description": "Action timeout in seconds. Action will get killed if it doesn't finish in timeout seconds.",
        "_name": "timeout"
      },
      "env": {
        "type": "object",
        "description": "Environment variables which will be available to the script.",
        "_name": "env"
      }
    },
    "id": "5cc678336cb8de3b0e25977d",
    "name": "python-script"
  },
  "elapsed_seconds": 0.477576,
  "web_url": "https://stackstorm/#/history/5ccce5796cb8de68daba91ab/general",
  "result": {
    "result": true,
    "exit_code": 0,
    "stderr": "",
    "stdout": ""
  },
  "context": {
    "re-run": {
      "ref": "5ccce4d26cb8de3b45c2b63e"
    },
    "trace_context": {
      "id_": "5ccce4d26cb8de3b45c2b639"
    },
    "user": "st2admin",
    "pack": "sensu"
  },
  "action": {
    "description": "Silence a Sensu client or check",
    "runner_type": "python-script",
    "tags": [],
    "enabled": true,
    "metadata_file": "actions/silence.yaml",
    "pack": "sensu",
    "entry_point": "silence.py",
    "notify": {},
    "output_schema": {},
    "uid": "action:sensu:silence",
    "parameters": {
      "message": {
        "default": "Stash created by StackStorm",
        "type": "string",
        "description": "Message to post with the silence stash.",
        "_name": "message"
      },
      "client": {
        "required": true,
        "type": "string",
        "description": "Client to silence.",
        "_name": "client"
      },
      "check": {
        "type": "string",
        "description": "The optional check to silence on the specified client.",
        "_name": "check"
      },
      "expiration": {
        "type": "integer",
        "description": "Time to expire (in seconds).  If blank, stash will not expire.",
        "_name": "expiration"
      }
    },
    "ref": "sensu.silence",
    "id": "5cc7ceb66cb8de119cdde16f",
    "name": "silence"
  },
  "liveaction": {
    "runner_info": {
      "hostname": "stackstorm",
      "pid": 16161
    },
    "parameters": {
      "message": "arun",
      "client": "xyz",
      "log_level": "DEBUG",
      "check": "ha_status"
    },
    "action_is_workflow": false,
    "callback": {},
    "action": "sensu.silence",
    "id": "5ccce5796cb8de68daba91aa"
  },
  "id": "5ccce5796cb8de68daba91ab",
  "end_timestamp": "2019-05-04T01:06:01.962915Z"
}
(Tomaz Muraus) #2

It could be a lot of things and it’s hard to say without seeing the logs (do other sensu pack actions works? it could be a misconfigured sensu pack, sensu api version issue, etc).

I would try to manually run sensu.silence action and while doing that, inspect StackStorm action runner (/var/log/st2/st2actionrunner*.log) and sensu log files (/var/log/sensu/*).

It could also be related to this issue - silence action does not work with sensu v0.29+ · Issue #7 · StackStorm-Exchange/stackstorm-sensu · GitHub.

(Arunn Srinivasan) #3

@kami
Thanks for the reply and pointer, yes i also stumbled on the v0.29+, i have added a comment in the Issue #7 section, i changed the silence action code and was able to silence it, verified thru the sensu uchiwa ui aswell.
Will do some testing and then will submit a PR.

thx