Jira Metrics – using Elasticsearch
Part 1 –
By Chadwiki
How to index Jira data into ElasticSearch? and Why?
Because: I wanted to Index Jira Incident data for Managing the QoS of Services, while prioritizing improvements and communicate out to the teams.
Some have asked why I chose to use a river and not write my own code. Because I don’t mind supporting the work or help improving someone else’s . Bottom line: I also don’t need to support or distract may with anything else.
Versions: Elasticsearch 1.4.0 & Kibana3
Install the River Plugin: https://github.com/searchisko/elasticsearch-river-jira
bin/plugin -url https://repository.jboss.org/nexus/content/groups/public-jboss/org/jboss/elasticsearch/elasticsearch-river-jira/1.8.4/elasticsearch-river-jira-1.8.4.zip -install elasticsearch-river-jira
Start:
Create your Index:
curl -XPUT 'http://localhost:9200/_river'
curl -XPUT 'http://localhost:9200/jira_river_activity'curl -XPUT 'http://localhost:9200/jira'
Post your Mapping for the data/Project:
curl -XPUT localhost:9200/jira/jira_issue/_mapping -d '
{
"jira_issue" : {
"_timestamp" : { "enabled" : true },
"properties" : {
"project_key" : {"type" : "string", "analyzer" : "keyword"},
"source" : {"type" : "string", "analyzer" : "keyword"}
}
}
}
'Post your Mappings for Comments and ChangeLogs data: (I’m using Child)
curl -XPUT localhost:9200/jira/jira_issue_comment/_mapping -d '
{
"jira_issue_comment" : {
"_timestamp" : { "enabled" : true },
"properties" : {
"project_key" : {"type" : "string", "analyzer" : "keyword"},
"source" : {"type" : "string", "analyzer" : "keyword"}
}
}
}
'
curl -XPUT localhost:9200/jira/jira_issue_change/_mapping -d '
{
"jira_issue_change" : {
"_timestamp" : { "enabled" : true },
"properties" : {
"project_key" : {"type" : "string", "analyzer" : "keyword"},
"source" : {"type" : "string", "analyzer" : "keyword"}
}
}
}
'Creating your River Config for Jira:
I will follow this with my actual config plus the custom jira fields – to discover these fields and their names, export the ticket as XML.
curl -XPUT localhost:9200/_river/jira/_meta -d '
{
"type" : "jira",
"jira" : {
"urlBase" : "https://issues.jboss.org",
"username" : "jira_username",
"pwd" : "jira_user_password",
"jqlTimeZone" : "America/New York",
"timeout" : "5s",
"maxIssuesPerRequest" : 50,
"projectKeysIndexed" : "ORG,AS7",
"indexUpdatePeriod" : "5m",
"indexFullUpdatePeriod" : "1h",
"maxIndexingThreads" : 2
"jqlTempate" : "project='%s'%s%s ORDER BY updated ASC"
},
"index" : {
"index" : "my_jira_index",
"type" : "jira_issue"
},
"activity_log": {
"index" : "jira_river_activity",
"type" : "jira_river_indexupdate"
}
}
My Custom Mapping for my data this should be applied before data ingestion / river creation:
curl -XPUT localhost:9200/jira/_mapping/incident -d '
{
"incident": {
"properties": {
"action_to_resolve": {
"type": "string"
},
"actual_end": {
"type": "date",
"format": "dateOptionalTime"
},
"actual_outage": {
"type": "long"
},
"actual_start": {
"type": "date",
"format": "dateOptionalTime"
},
"affected_ci": {
"type": "string"
},
"affected_service": {
"type": "string"
},
"affected_services": {
"type": "string",
"index": "not_analyzed"
},
"assignee": {
"properties": {
"display_name": {
"type": "string"
},
"email_address": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"bridge_id": {
"type": "string"
},
"browsers": {
"type": "string"
},
"calculated_risk": {
"type": "string"
},
"changelogs": {
"properties": {
"change_author": {
"properties": {
"display_name": {
"type": "string"
},
"email_address": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"change_created": {
"type": "date",
"format": "dateOptionalTime"
},
"change_id": {
"type": "string"
},
"change_items": {
"properties": {
"field": {
"type": "string"
},
"fieldtype": {
"type": "string"
},
"from": {
"type": "string"
},
"fromString": {
"type": "string"
},
"to": {
"type": "string"
},
"toString": {
"type": "string"
}
}
},
"document_url": {
"type": "string"
}
}
},
"closed_by": {
"type": "string"
},
"comments": {
"properties": {
"comment_author": {
"properties": {
"display_name": {
"type": "string"
},
"email_address": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"comment_body": {
"type": "string"
},
"comment_created": {
"type": "date",
"format": "dateOptionalTime"
},
"comment_id": {
"type": "string"
},
"comment_updated": {
"type": "date",
"format": "dateOptionalTime"
},
"comment_updater": {
"properties": {
"display_name": {
"type": "string"
},
"email_address": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"document_url": {
"type": "string"
}
}
},
"components": {
"type": "string",
"index": "not_analyzed",
"fields": {
"tok": {
"type": "string",
"index": "analyzed"
}
}
},
"created": {
"type": "date",
"format": "dateOptionalTime"
},
"datacenter": {
"type": "string",
"index": "not_analyzed"
},
"db_impact": {
"type": "string"
},
"description": {
"type": "string"
},
"doc_ui_impact": {
"type": "string"
},
"document_url": {
"type": "string"
},
"fix_versions": {
"properties": {
"archived": {
"type": "boolean"
},
"description": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"releaseDate": {
"type": "date",
"format": "dateOptionalTime"
},
"released": {
"type": "boolean"
},
"self": {
"type": "string"
}
}
},
"incident_detection": {
"type": "string"
},
"incident_diagnosis": {
"type": "string"
},
"incident_resolution": {
"type": "string"
},
"incident_source": {
"type": "string"
},
"issue_key": {
"type": "string"
},
"issue_type": {
"type": "string",
"index": "not_analyzed"
},
"issuelinks": {
"properties": {
"id": {
"type": "string"
},
"inwardIssue": {
"properties": {
"fields": {
"properties": {
"issuetype": {
"properties": {
"description": {
"type": "string"
},
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
},
"subtask": {
"type": "boolean"
}
}
},
"priority": {
"properties": {
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"status": {
"properties": {
"description": {
"type": "string"
},
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"summary": {
"type": "string"
}
}
},
"id": {
"type": "string"
},
"key": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"outwardIssue": {
"properties": {
"fields": {
"properties": {
"issuetype": {
"properties": {
"description": {
"type": "string"
},
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
},
"subtask": {
"type": "boolean"
}
}
},
"priority": {
"properties": {
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"status": {
"properties": {
"description": {
"type": "string"
},
"iconUrl": {
"type": "string"
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"summary": {
"type": "string"
}
}
},
"id": {
"type": "string"
},
"key": {
"type": "string"
},
"self": {
"type": "string"
}
}
},
"self": {
"type": "string"
},
"type": {
"properties": {
"id": {
"type": "string"
},
"inward": {
"type": "string"
},
"name": {
"type": "string"
},
"outward": {
"type": "string"
},
"self": {
"type": "string"
}
}
}
}
},
"key_findings": {
"type": "string"
},
"known_issue": {
"type": "string"
},
"labels": {
"type": "string"
},
"method_notification": {
"type": "string",
"index": "not_analyzed"
},
"project_key": {
"type": "string"
},
"project_name": {
"type": "string"
},
"rank": {
"type": "string"
},
"reporter": {
"properties": {
"display_name": {
"type": "string"
},
"email_address": {
"type": "string"
},
"username": {
"type": "string"
}
}
},
"resolution": {
"type": "string"
},
"resolutiondate": {
"type": "date",
"format": "dateOptionalTime"
},
"root_cause": {
"type": "string"
},
"source": {
"type": "string"
},
"state": {
"type": "string"
},
"status": {
"type": "string"
},
"summary": {
"type": "string"
},
"time_in_status": {
"type": "string"
},
"time_slot": {
"type": "string",
"index": "not_analyzed"
},
"updated": {
"type": "date",
"format": "dateOptionalTime"
},
"urgency": {
"type": "string"
},
"work_around_notes": {
"type": "string"
}
}
}
}
Hear is the actual config with custom fields:
curl -XPUT localhost:9200/_river/jira/_meta -d '
{
"type": "jira",
"jira": {
"urlBase": "https://jira.mycompany.com/jira",
"username": "sa_usernamer",
"pwd": "Sa_UserPassword",
"timeout": "50s",
"maxIssuesPerRequest": 250,
"projectKeysIndexed": "OPI",
"indexUpdatePeriod": "10m",
"indexFullUpdatePeriod": "0",
"maxIndexingThreads": 2
},
"index": {
"index": "jira",
"type": "incident",
"fields": {
"project_name": {
"jira_field": "fields.project.name"
},
"issue_type": {
"jira_field": "fields.issuetype.name"
},
"issuelinks": {
"jira_field": "fields.issuelinks"
},
"summary": {
"jira_field": "fields.summary"
},
"status": {
"jira_field": "fields.status.name"
},
"created": {
"jira_field": "fields.created"
},
"updated": {
"jira_field": "fields.updated"
},
"resolution": {
"jira_field": "fields.resolution.name"
},
"resolutiondate": {
"jira_field": "fields.resolutiondate"
},
"description": {
"jira_field": "fields.description"
},
"labels": {
"jira_field": "fields.labels"
},
"reporter": {
"jira_field": "fields.reporter",
"value_filter": "user"
},
"assignee": {
"jira_field": "fields.assignee",
"value_filter": "user"
},
"fix_versions": {
"jira_field": "fields.fixVersions"
},
"components": {
"jira_field": "fields.components.name"
},
"actual_start": {
"jira_field": "fields.customfield_10842"
},
"actual_end": {
"jira_field": "fields.customfield_10844"
},
"urgency": {
"jira_field": "fields.customfield_11303.value"
},
"affected_ci": {
"jira_field": "fields.customfield_10809"
},
"affected_service": {
"jira_field": "fields.customfield_10840.value"
},
"affected_services": {
"jira_field": "fields.customfield_11301.value"
},
"bridge_id": {
"jira_field": "fields.customfield_11614.value"
},
"browsers": {
"jira_field": "fields.customfield_10140.value"
},
"calculated_risk": {
"jira_field": "fields.customfield_11613"
},
"closed_by": {
"jira_field": "fields.customfield_10674"
},
"db_impact": {
"jira_field": "fields.customfield_10021.value"
},
"datacenter": {
"jira_field": "fields.customfield_10836.value"
},
"doc_ui_impact": {
"jira_field": "fields.customfield_10020.value"
},
"incident_detection": {
"jira_field": "fields.customfield_10850.value"
},
"incident_diagnosis": {
"jira_field": "fields.customfield_10851.value"
},
"incident_resolution": {
"jira_field": "fields.customfield_10852.value"
},
"incident_source": {
"jira_field": "fields.customfield_10808.value"
},
"key_findings": {
"jira_field": "fields.customfield_10880"
},
"known_issue": {
"jira_field": "fields.customfield_11504.value"
},
"method_notification": {
"jira_field": "fields.customfield_10811.value"
},
"rank": {
"jira_field": "fields.customfield_11000"
},
"root_cause": {
"jira_field": "fields.customfield_11400"
},
"severity": {
"jira_field": "fields.customfield_10010"
},
"state": {
"jira_field": "fields.customfield_10846.value"
},
"time_in_status": {
"jira_field": "fields.customfield_10523"
},
"work_around_notes": {
"jira_field": "fields.customfield_10845"
},
"action_to_resolve": {
"jira_field": "fields.customfield_12003.value"
}
},
"value_filters": {
"user": {
"name": "username",
"emailAddress": "email_address",
"displayName": "display_name"
},
"name": {
"name": "name"
}
},
"comment_mode": "embedded",
"comment_type": "jira_issue_comment",
"field_comments": "comments",
"comment_fields": {
"comment_id": {
"jira_field": "id"
},
"comment_body": {
"jira_field": "body"
},
"comment_author": {
"jira_field": "author",
"value_filter": "user"
},
"comment_updater": {
"jira_field": "updateAuthor",
"value_filter": "user"
},
"comment_created": {
"jira_field": "created"
},
"comment_updated": {
"jira_field": "updated"
}
},
"changelog_mode": "embedded",
"field_changelogs": "changelogs",
"changelog_fields": {
"change_id": {
"jira_field": "id"
},
"change_items": {
"jira_field": "items"
},
"change_author": {
"jira_field": "author",
"value_filter": "user"
},
"change_created": {
"jira_field": "created"
}
}
},
"activity_log": {
"index": "jira_river_activity",
"type": "jira_river_indexupdate"
}
}
'Check your River Status:
curl -XGET localhost:9200/_river/jira/_mgm_jr/stateNext steps: Check your data within Elasticsearch & Visualize it using Kibana