Jira Metrics – using Elasticsearch

Part 1 – 

By Chadwiki

How to index Jira data into ElasticSearch? and Why?

Because: I wanted to Index Jira Incident data for Managing the QoS of Services, while prioritizing improvements and communicate out to the teams.

Some have asked why I chose to use a river and not write my own code. Because I don’t mind supporting the work or help improving someone else’s .  Bottom line:  I also don’t need to support or distract may with anything else.

Versions: Elasticsearch 1.4.0 & Kibana3

Install the River Plugin: https://github.com/searchisko/elasticsearch-river-jira

bin/plugin -url https://repository.jboss.org/nexus/content/groups/public-jboss/org/jboss/elasticsearch/elasticsearch-river-jira/1.8.4/elasticsearch-river-jira-1.8.4.zip -install elasticsearch-river-jira

Start:

Create your Index:

curl -XPUT 'http://localhost:9200/_river'
curl -XPUT 'http://localhost:9200/jira_river_activity'
curl -XPUT 'http://localhost:9200/jira'

Post your Mapping for the data/Project:

curl -XPUT localhost:9200/jira/jira_issue/_mapping -d '
{
    "jira_issue" : {
        "_timestamp" : { "enabled" : true },
        "properties" : {
            "project_key" : {"type" : "string", "analyzer" : "keyword"},
            "source"      : {"type" : "string", "analyzer" : "keyword"}
        }
    }
}
'

Post your Mappings for Comments and ChangeLogs data: (I’m using Child)

curl -XPUT localhost:9200/jira/jira_issue_comment/_mapping -d '
{
    "jira_issue_comment" : {
        "_timestamp" : { "enabled" : true },
        "properties" : {
            "project_key" : {"type" : "string", "analyzer" : "keyword"},
            "source"      : {"type" : "string", "analyzer" : "keyword"}
        }
    }
}
'

curl -XPUT localhost:9200/jira/jira_issue_change/_mapping -d '
{
    "jira_issue_change" : {
        "_timestamp" : { "enabled" : true },
        "properties" : {
            "project_key" : {"type" : "string", "analyzer" : "keyword"},
            "source"      : {"type" : "string", "analyzer" : "keyword"}
        }
    }
}
'

Creating your River Config for Jira:

I will follow this with my actual config plus the custom jira fields – to discover these fields and their names, export the ticket as XML.

curl -XPUT localhost:9200/_river/jira/_meta -d '
{
    "type" : "jira",
    "jira" : {
        "urlBase"               : "https://issues.jboss.org",
        "username"              : "jira_username",
        "pwd"                   : "jira_user_password",
        "jqlTimeZone"           : "America/New York",
        "timeout"               : "5s",
        "maxIssuesPerRequest"   : 50,
        "projectKeysIndexed"    : "ORG,AS7",
        "indexUpdatePeriod"     : "5m",
        "indexFullUpdatePeriod" : "1h",
        "maxIndexingThreads"    : 2
        "jqlTempate"            : "project='%s'%s%s ORDER BY updated ASC"
    },
    "index" : {
        "index" : "my_jira_index",
        "type"  : "jira_issue"
    },
    "activity_log": {
        "index" : "jira_river_activity",
        "type"  : "jira_river_indexupdate"
    }
}

My Custom Mapping for my data this should be applied before data ingestion / river creation:

curl -XPUT localhost:9200/jira/_mapping/incident -d '
{
  "incident": {
    "properties": {
      "action_to_resolve": {
        "type": "string"
      },
      "actual_end": {
        "type": "date",
        "format": "dateOptionalTime"
      },
      "actual_outage": {
        "type": "long"
      },
      "actual_start": {
        "type": "date",
        "format": "dateOptionalTime"
      },
      "affected_ci": {
        "type": "string"
      },
      "affected_service": {
        "type": "string"
      },
      "affected_services": {
        "type": "string",
        "index": "not_analyzed"
      },
      "assignee": {
        "properties": {
          "display_name": {
            "type": "string"
          },
          "email_address": {
            "type": "string"
          },
          "username": {
            "type": "string"
          }
        }
      },
      "bridge_id": {
        "type": "string"
      },
      "browsers": {
        "type": "string"
      },
      "calculated_risk": {
        "type": "string"
      },
      "changelogs": {
        "properties": {
          "change_author": {
            "properties": {
              "display_name": {
                "type": "string"
              },
              "email_address": {
                "type": "string"
              },
              "username": {
                "type": "string"
              }
            }
          },
          "change_created": {
            "type": "date",
            "format": "dateOptionalTime"
          },
          "change_id": {
            "type": "string"
          },
          "change_items": {
            "properties": {
              "field": {
                "type": "string"
              },
              "fieldtype": {
                "type": "string"
              },
              "from": {
                "type": "string"
              },
              "fromString": {
                "type": "string"
              },
              "to": {
                "type": "string"
              },
              "toString": {
                "type": "string"
              }
            }
          },
          "document_url": {
            "type": "string"
          }
        }
      },
      "closed_by": {
        "type": "string"
      },
      "comments": {
        "properties": {
          "comment_author": {
            "properties": {
              "display_name": {
                "type": "string"
              },
              "email_address": {
                "type": "string"
              },
              "username": {
                "type": "string"
              }
            }
          },
          "comment_body": {
            "type": "string"
          },
          "comment_created": {
            "type": "date",
            "format": "dateOptionalTime"
          },
          "comment_id": {
            "type": "string"
          },
          "comment_updated": {
            "type": "date",
            "format": "dateOptionalTime"
          },
          "comment_updater": {
            "properties": {
              "display_name": {
                "type": "string"
              },
              "email_address": {
                "type": "string"
              },
              "username": {
                "type": "string"
              }
            }
          },
          "document_url": {
            "type": "string"
          }
        }
      },
      "components": {
        "type": "string",
        "index": "not_analyzed",
        "fields": {
          "tok": {
            "type": "string",
            "index": "analyzed"
          }
        }
      },
      "created": {
        "type": "date",
        "format": "dateOptionalTime"
      },
      "datacenter": {
        "type": "string",
        "index": "not_analyzed"
      },
      "db_impact": {
        "type": "string"
      },
      "description": {
        "type": "string"
      },
      "doc_ui_impact": {
        "type": "string"
      },
      "document_url": {
        "type": "string"
      },
      "fix_versions": {
        "properties": {
          "archived": {
            "type": "boolean"
          },
          "description": {
            "type": "string"
          },
          "id": {
            "type": "string"
          },
          "name": {
            "type": "string"
          },
          "releaseDate": {
            "type": "date",
            "format": "dateOptionalTime"
          },
          "released": {
            "type": "boolean"
          },
          "self": {
            "type": "string"
          }
        }
      },
      "incident_detection": {
        "type": "string"
      },
      "incident_diagnosis": {
        "type": "string"
      },
      "incident_resolution": {
        "type": "string"
      },
      "incident_source": {
        "type": "string"
      },
      "issue_key": {
        "type": "string"
      },
      "issue_type": {
        "type": "string",
        "index": "not_analyzed"
      },
      "issuelinks": {
        "properties": {
          "id": {
            "type": "string"
          },
          "inwardIssue": {
            "properties": {
              "fields": {
                "properties": {
                  "issuetype": {
                    "properties": {
                      "description": {
                        "type": "string"
                      },
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      },
                      "subtask": {
                        "type": "boolean"
                      }
                    }
                  },
                  "priority": {
                    "properties": {
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      }
                    }
                  },
                  "status": {
                    "properties": {
                      "description": {
                        "type": "string"
                      },
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      }
                    }
                  },
                  "summary": {
                    "type": "string"
                  }
                }
              },
              "id": {
                "type": "string"
              },
              "key": {
                "type": "string"
              },
              "self": {
                "type": "string"
              }
            }
          },
          "outwardIssue": {
            "properties": {
              "fields": {
                "properties": {
                  "issuetype": {
                    "properties": {
                      "description": {
                        "type": "string"
                      },
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      },
                      "subtask": {
                        "type": "boolean"
                      }
                    }
                  },
                  "priority": {
                    "properties": {
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      }
                    }
                  },
                  "status": {
                    "properties": {
                      "description": {
                        "type": "string"
                      },
                      "iconUrl": {
                        "type": "string"
                      },
                      "id": {
                        "type": "string"
                      },
                      "name": {
                        "type": "string"
                      },
                      "self": {
                        "type": "string"
                      }
                    }
                  },
                  "summary": {
                    "type": "string"
                  }
                }
              },
              "id": {
                "type": "string"
              },
              "key": {
                "type": "string"
              },
              "self": {
                "type": "string"
              }
            }
          },
          "self": {
            "type": "string"
          },
          "type": {
            "properties": {
              "id": {
                "type": "string"
              },
              "inward": {
                "type": "string"
              },
              "name": {
                "type": "string"
              },
              "outward": {
                "type": "string"
              },
              "self": {
                "type": "string"
              }
            }
          }
        }
      },
      "key_findings": {
        "type": "string"
      },
      "known_issue": {
        "type": "string"
      },
      "labels": {
        "type": "string"
      },
      "method_notification": {
        "type": "string",
        "index": "not_analyzed"
      },
      "project_key": {
        "type": "string"
      },
      "project_name": {
        "type": "string"
      },
      "rank": {
        "type": "string"
      },
      "reporter": {
        "properties": {
          "display_name": {
            "type": "string"
          },
          "email_address": {
            "type": "string"
          },
          "username": {
            "type": "string"
          }
        }
      },
      "resolution": {
        "type": "string"
      },
      "resolutiondate": {
        "type": "date",
        "format": "dateOptionalTime"
      },
      "root_cause": {
        "type": "string"
      },
      "source": {
        "type": "string"
      },
      "state": {
        "type": "string"
      },
      "status": {
        "type": "string"
      },
      "summary": {
        "type": "string"
      },
      "time_in_status": {
        "type": "string"
      },
      "time_slot": {
        "type": "string",
        "index": "not_analyzed"
      },
      "updated": {
        "type": "date",
        "format": "dateOptionalTime"
      },
      "urgency": {
        "type": "string"
      },
      "work_around_notes": {
        "type": "string"
      }
    }
  }
}

Hear is the actual config with custom fields:

curl -XPUT localhost:9200/_river/jira/_meta -d '
{
  "type": "jira",
  "jira": {
    "urlBase": "https://jira.mycompany.com/jira",
    "username": "sa_usernamer",
    "pwd": "Sa_UserPassword",
    "timeout": "50s",
    "maxIssuesPerRequest": 250,
    "projectKeysIndexed": "OPI",
    "indexUpdatePeriod": "10m",
    "indexFullUpdatePeriod": "0",
    "maxIndexingThreads": 2
  },
  "index": {
    "index": "jira",
    "type": "incident",
    "fields": {
      "project_name": {
        "jira_field": "fields.project.name"
      },
      "issue_type": {
        "jira_field": "fields.issuetype.name"
      },
      "issuelinks": {
        "jira_field": "fields.issuelinks"
      },
      "summary": {
        "jira_field": "fields.summary"
      },
      "status": {
        "jira_field": "fields.status.name"
      },
      "created": {
        "jira_field": "fields.created"
      },
      "updated": {
        "jira_field": "fields.updated"
      },
      "resolution": {
        "jira_field": "fields.resolution.name"
      },
      "resolutiondate": {
        "jira_field": "fields.resolutiondate"
      },
      "description": {
        "jira_field": "fields.description"
      },
      "labels": {
        "jira_field": "fields.labels"
      },
      "reporter": {
        "jira_field": "fields.reporter",
        "value_filter": "user"
      },
      "assignee": {
        "jira_field": "fields.assignee",
        "value_filter": "user"
      },
      "fix_versions": {
        "jira_field": "fields.fixVersions"
      },
      "components": {
        "jira_field": "fields.components.name"
      },
      "actual_start": {
        "jira_field": "fields.customfield_10842"
      },
      "actual_end": {
        "jira_field": "fields.customfield_10844"
      },
      "urgency": {
        "jira_field": "fields.customfield_11303.value"
      },
      "affected_ci": {
        "jira_field": "fields.customfield_10809"
      },
      "affected_service": {
        "jira_field": "fields.customfield_10840.value"
      },
      "affected_services": {
        "jira_field": "fields.customfield_11301.value"
      },
      "bridge_id": {
        "jira_field": "fields.customfield_11614.value"
      },
      "browsers": {
        "jira_field": "fields.customfield_10140.value"
      },
      "calculated_risk": {
        "jira_field": "fields.customfield_11613"
      },
      "closed_by": {
        "jira_field": "fields.customfield_10674"
      },
      "db_impact": {
        "jira_field": "fields.customfield_10021.value"
      },
      "datacenter": {
        "jira_field": "fields.customfield_10836.value"
      },
      "doc_ui_impact": {
        "jira_field": "fields.customfield_10020.value"
      },
      "incident_detection": {
        "jira_field": "fields.customfield_10850.value"
      },
      "incident_diagnosis": {
        "jira_field": "fields.customfield_10851.value"
      },
      "incident_resolution": {
        "jira_field": "fields.customfield_10852.value"
      },
      "incident_source": {
        "jira_field": "fields.customfield_10808.value"
      },
      "key_findings": {
        "jira_field": "fields.customfield_10880"
      },
      "known_issue": {
        "jira_field": "fields.customfield_11504.value"
      },
      "method_notification": {
        "jira_field": "fields.customfield_10811.value"
      },
      "rank": {
        "jira_field": "fields.customfield_11000"
      },
      "root_cause": {
        "jira_field": "fields.customfield_11400"
      },
      "severity": {
        "jira_field": "fields.customfield_10010"
      },
      "state": {
        "jira_field": "fields.customfield_10846.value"
      },
      "time_in_status": {
        "jira_field": "fields.customfield_10523"
      },
      "work_around_notes": {
        "jira_field": "fields.customfield_10845"
      },
      "action_to_resolve": {
        "jira_field": "fields.customfield_12003.value"
      }
    },
    "value_filters": {
      "user": {
        "name": "username",
        "emailAddress": "email_address",
        "displayName": "display_name"
      },
      "name": {
        "name": "name"
      }
    },
    "comment_mode": "embedded",
    "comment_type": "jira_issue_comment",
    "field_comments": "comments",
    "comment_fields": {
      "comment_id": {
        "jira_field": "id"
      },
      "comment_body": {
        "jira_field": "body"
      },
      "comment_author": {
        "jira_field": "author",
        "value_filter": "user"
      },
      "comment_updater": {
        "jira_field": "updateAuthor",
        "value_filter": "user"
      },
      "comment_created": {
        "jira_field": "created"
      },
      "comment_updated": {
        "jira_field": "updated"
      }
    },
    "changelog_mode": "embedded",
    "field_changelogs": "changelogs",
    "changelog_fields": {
      "change_id": {
        "jira_field": "id"
      },
      "change_items": {
        "jira_field": "items"
      },
      "change_author": {
        "jira_field": "author",
        "value_filter": "user"
      },
      "change_created": {
        "jira_field": "created"
      }
    }
  },
  "activity_log": {
    "index": "jira_river_activity",
    "type": "jira_river_indexupdate"
  }
}
'

Check your River Status:

curl -XGET localhost:9200/_river/jira/_mgm_jr/state

Next steps:  Check your data within Elasticsearch & Visualize it using Kibana