[es]성능체크

카테고리를 파이썬으로보내야 하나..

애매하네

 

시간이 없어 급하게 작업하다 보니 문제가 생김

ecs 가 재실행하면서 노드의 키값이 바뀌다 보니... 죽을때 마다 매번 노가다를

 

import json
import time

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from ssl import create_default_context
import matplotlib.pyplot as plt
from matplotlib.collections import EventCollection
import numpy as np
from hurry.filesize import size

import threading
import time

import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
plt.rcParams['font.family'] = 'AppleGothic'

def indices_cat():
    data = client.nodes.stats()
    return data

if __name__ == '__main__':
    client = Elasticsearch("https://elastic:{암오}@{URL}:443/", ca_certs=False,
                           verify_certs=False)
    Y_LABEL = "사용률(%)"
    X_LABEL = "회/s"
    #반복
    RANGE = 600
    #Y 축
    Y_LIMIT = 100
    print("------------------------------ start ------------------------------")
    os_mem_coodinate_95_used_percent = []
    os_cpu_coodinate_95_percent = []
    jvm_mem_coodinate_95_heap_used_percent = []

    # Mdh0AU5LT46D8C3XSEY5wQ
    os_mem_coodinate_22151_used_percent = []
    os_cpu_coodinate_22151_percent = []
    jvm_mem_coodinate_22151_heap_used_percent = []

    # ZFgpRG-cSwyuLOOEKCeALg
    os_mem_coodinate_105_used_percent = []
    os_cpu_coodinate_105_percent = []
    jvm_mem_coodinate_105_heap_used_percent = []

    # RmunxwPLQvqn4fQpShLtGQ
    os_mem_coodinate_121_used_percent = []
    os_cpu_coodinate_121_percent = []
    jvm_mem_coodinate_121_heap_used_percent = []

    # Vo7OJccYSBizopcrlGj7MQ
    os_mem_master_111_used_percent = []
    os_cpu_master_111_percent = []
    jvm_mem_master_111_heap_used_percent = []

    # pJJTt3JFS2q-w1iSiuWDpA
    os_mem_coodinate_120_used_percent = []
    os_cpu_coodinate_120_percent = []
    jvm_mem_coodinate_120_heap_used_percent = []

    # Braze
    os_mem_coodinate_112_used_percent = []
    os_cpu_coodinate_112_percent = []
    jvm_mem_coodinate_112_heap_used_percent = []

    # eUUFNyGzTD2H8MOXASwI8w
    os_mem_data_106_used_percent = []
    os_cpu_data_106_percent = []
    jvm_mem_data_106_heap_used_percent = []

    # HZ8YO6AwSQifVzrCnvGhYA
    os_mem_data_123_used_percent = []
    os_cpu_data_123_percent = []
    jvm_mem_data_123_heap_used_percent = []

    # E4rJ_JeDR2u6i3MSzjNe5Q
    os_mem_coodinate_151_used_percent = []
    os_cpu_coodinate_151_percent = []
    jvm_mem_coodinate_151_heap_used_percent = []

    # AVDPaek-RfafVm11rI0liQ
    os_mem_data_125_used_percent = []
    os_cpu_data_125_percent = []
    jvm_mem_data_125_heap_used_percent = []

    for i in range(0, RANGE):
        try:
            print(i)
            time.sleep(1)
            response = indices_cat()

            coodinate_95 = response['nodes']['I0WmrPiWS4-PH863oSJ_sg'] #
            coodinate_22151 = response['nodes']['8gn7KDTDQRGDW6RSArwPpw'] #braze
            coodinate_105 = response['nodes']['Mdh0AU5LT46D8C3XSEY5wQ']
            coodinate_121 = response['nodes']['pJJTt3JFS2q-w1iSiuWDpA'] #
            master_111 = response['nodes']['Vo7OJccYSBizopcrlGj7MQ'] #
            coodinate_120 = response['nodes']['RmunxwPLQvqn4fQpShLtGQ'] #
            coodinate_112 = response['nodes']['rqY3qvGvRGS2nVg_eygJAQ'] #
            data_106 = response['nodes']['HZ8YO6AwSQifVzrCnvGhYA'] #
            data_123 = response['nodes']['eUUFNyGzTD2H8MOXASwI8w'] #
            coodinate_151 = response['nodes']['PYLJNXjHQhyuES7NxF9_wA']
            data_125 = response['nodes']['AVDPaek-RfafVm11rI0liQ'] #

            os_mem_coodinate_95_used_percent.append(coodinate_95['os']['mem']['used_percent'])
            os_cpu_coodinate_95_percent.append(coodinate_95['os']['cpu']['percent'])
            jvm_mem_coodinate_95_heap_used_percent.append(coodinate_95['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_22151_used_percent.append(coodinate_22151['os']['mem']['used_percent'])
            os_cpu_coodinate_22151_percent.append(coodinate_22151['os']['cpu']['percent'])
            jvm_mem_coodinate_22151_heap_used_percent.append(coodinate_22151['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_105_used_percent.append(coodinate_105['os']['mem']['used_percent'])
            os_cpu_coodinate_105_percent.append(coodinate_105['os']['cpu']['percent'])
            jvm_mem_coodinate_105_heap_used_percent.append(coodinate_105['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_121_used_percent.append(coodinate_121['os']['mem']['used_percent'])
            os_cpu_coodinate_121_percent.append(coodinate_121['os']['cpu']['percent'])
            jvm_mem_coodinate_121_heap_used_percent.append(coodinate_121['jvm']['mem']['heap_used_percent'])

            os_mem_master_111_used_percent.append(master_111['os']['mem']['used_percent'])
            os_cpu_master_111_percent.append(master_111['os']['cpu']['percent'])
            jvm_mem_master_111_heap_used_percent.append(master_111['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_120_used_percent.append(coodinate_120['os']['mem']['used_percent'])
            os_cpu_coodinate_120_percent.append(coodinate_120['os']['cpu']['percent'])
            jvm_mem_coodinate_120_heap_used_percent.append(coodinate_120['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_112_used_percent.append(coodinate_112['os']['mem']['used_percent'])
            os_cpu_coodinate_112_percent.append(coodinate_112['os']['cpu']['percent'])
            jvm_mem_coodinate_112_heap_used_percent.append(coodinate_112['jvm']['mem']['heap_used_percent'])

            os_mem_data_106_used_percent.append(data_106['os']['mem']['used_percent'])
            os_cpu_data_106_percent.append(data_106['os']['cpu']['percent'])
            jvm_mem_data_106_heap_used_percent.append(data_106['jvm']['mem']['heap_used_percent'])

            os_mem_data_123_used_percent.append(data_123['os']['mem']['used_percent'])
            os_cpu_data_123_percent.append(data_123['os']['cpu']['percent'])
            jvm_mem_data_123_heap_used_percent.append(data_123['jvm']['mem']['heap_used_percent'])

            os_mem_coodinate_151_used_percent.append(coodinate_151['os']['mem']['used_percent'])
            os_cpu_coodinate_151_percent.append(coodinate_151['os']['cpu']['percent'])
            jvm_mem_coodinate_151_heap_used_percent.append(coodinate_151['jvm']['mem']['heap_used_percent'])

            os_mem_data_125_used_percent.append(data_125['os']['mem']['used_percent'])
            os_cpu_data_125_percent.append(data_125['os']['cpu']['percent'])
            jvm_mem_data_125_heap_used_percent.append(data_125['jvm']['mem']['heap_used_percent'])
        except KeyError:
            break

    xdata = range(len(os_mem_coodinate_95_used_percent))

    #--------------------------------------------

    ydata1 = os_mem_master_111_used_percent

    fig = plt.figure(figsize=(15, 10))
    fig.subplots_adjust(left=0.125, bottom=0.1,  right=0.9, top=0.9, wspace=0.2, hspace=0.35)

    ax = fig.add_subplot(3, 3, 1)
    ax.plot(xdata, ydata1, color='tab:blue')
    ax.set_ylabel(Y_LABEL)
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE -1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('OS memory used rate - master')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 2)

    # create some y data points
    ydata1 = jvm_mem_master_111_heap_used_percent
    ax.plot(xdata, ydata1, color='tab:blue')
    ax.set_xlabel(X_LABEL)

    # set the limits
    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('Heap memory used rate - master')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 3)
    ydata1 = os_cpu_master_111_percent
    ax.plot(xdata, ydata1, color='tab:blue')
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('OS cpu used rate - master')
    #--------------------------------------------
    #--------------------------------------------

    ydata1 = os_mem_data_106_used_percent
    ydata2 = os_mem_data_123_used_percent
    ydata3 = os_mem_data_125_used_percent

    # fig = plt.figure(figsize=(10, 15))
    ax = fig.add_subplot(3, 3, 4)

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:orange')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.set_ylabel(Y_LABEL)
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE -1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('OS memory used rate - data')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 5)

    ydata1 = jvm_mem_data_106_heap_used_percent
    ydata2 = jvm_mem_data_123_heap_used_percent
    ydata3 = jvm_mem_data_125_heap_used_percent

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:orange')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('Heap memory used rate - data')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 6)

    ydata1 = os_cpu_data_106_percent
    ydata2 = os_cpu_data_123_percent
    ydata3 = os_cpu_data_125_percent

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:orange')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('OS cpu used rate - data')
    #--------------------------------------------
    #--------------------------------------------

    #--------------------------------------------
    # create some y data points
    ydata1 = os_mem_coodinate_95_used_percent
    ydata2 = os_mem_coodinate_22151_used_percent
    ydata3 = os_mem_coodinate_105_used_percent
    ydata4 = os_mem_coodinate_121_used_percent
    ydata5 = os_mem_coodinate_120_used_percent
    ydata6 = os_mem_coodinate_112_used_percent
    ydata7 = os_mem_coodinate_151_used_percent

    # fig = plt.figure(figsize=(10, 15))
    ax = fig.add_subplot(3, 3, 7)

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:gray')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.plot(xdata, ydata4, color='tab:cyan')
    ax.plot(xdata, ydata5, color='tab:green')
    ax.plot(xdata, ydata6, color='tab:orange')
    ax.plot(xdata, ydata7, color='tab:purple')

    ax.set_ylabel(Y_LABEL)
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE -1])
    ax.set_ylim([0, Y_LIMIT])

    ax.set_title('OS memory used rate - coordi')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 8)

    # create some y data points
    ydata1 = jvm_mem_coodinate_95_heap_used_percent
    ydata2 = jvm_mem_coodinate_22151_heap_used_percent
    ydata3 = jvm_mem_coodinate_105_heap_used_percent
    ydata4 = jvm_mem_coodinate_121_heap_used_percent
    ydata5 = jvm_mem_coodinate_120_heap_used_percent
    ydata6 = jvm_mem_coodinate_112_heap_used_percent
    ydata7 = jvm_mem_coodinate_151_heap_used_percent

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:gray')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.plot(xdata, ydata4, color='tab:cyan')
    ax.plot(xdata, ydata5, color='tab:green')
    ax.plot(xdata, ydata6, color='tab:orange')
    ax.plot(xdata, ydata7, color='tab:purple')

    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('Heap memory used rate - coordi')

    #--------------------------------------------
    ax = fig.add_subplot(3, 3, 9)
    # create some y data points
    ydata1 = os_cpu_coodinate_95_percent
    ydata2 = os_cpu_coodinate_22151_percent
    ydata3 = os_cpu_coodinate_105_percent
    ydata4 = os_cpu_coodinate_121_percent
    ydata5 = os_cpu_coodinate_120_percent
    ydata6 = os_cpu_coodinate_112_percent
    ydata7 = os_cpu_coodinate_151_percent

    ax.plot(xdata, ydata1, color='tab:blue')
    ax.plot(xdata, ydata2, color='tab:gray')
    ax.plot(xdata, ydata3, color='tab:red')
    ax.plot(xdata, ydata4, color='tab:cyan')
    ax.plot(xdata, ydata5, color='tab:green')
    ax.plot(xdata, ydata6, color='tab:orange')
    ax.plot(xdata, ydata7, color='tab:purple')
    ax.set_xlabel(X_LABEL)

    ax.set_xlim([0, RANGE-1])
    ax.set_ylim([0, Y_LIMIT])
    ax.set_title('OS cpu used rate - coordi')
    #--------------------------------------------
    #--------------------------------------------
    # display the plot
    plt.show()

    print("------------------------------ end ------------------------------")

 

#!/bin/bash

cnt=0
while (("${cnt}" < 60)); do
  echo "${cnt}"
  nohup sh run.sh 2>&1 >braze_log/braze_"${cnt}"_$(date +%Y)-$(date +%m)-$(date +%d)_$(date +%H):$(date +%M):$(date +%S).txt &
  ((cnt = "${cnt}" + 1))
  sleep 1
done

echo "END"

#!/bin/bash

cnt=0
while (("${cnt}" < 800)); do
  echo "${cnt} data now : $(date +%Y)-$(date +%m)-$(date +%d) $(date +%H):$(date +%M):$(date +%S)"
  curl -X GET "localhost:9200/hyper-item/_search?pretty" -u elastic:elastic1! -H 'Content-Type: application/json' -d'
      {
        "from": 0,
        "size": 20,
        "timeout": "60s",
        "query": {
          "bool": {
            "filter": [
              {
                "range": {
                  "saleStartDt": {
                    "from": null,
                    "to": "now/m",
                    "include_lower": true,
                    "include_upper": true,
                    "boost": 1
                  }
                }
              },
              {
                "range": {
                  "saleEndDt": {
                    "from": "now/m",
                    "to": null,
                    "include_lower": true,
                    "include_upper": true,
                    "boost": 1
                  }
                }
              },
              {
                "term": {
                  "docDispYn": {
                    "value": "Y",
                    "boost": 1
                  }
                }
              },
              {
                "bool": {
                  "should": [
                    {
                      "terms": {
                        "itemStoreInfo.storeId": [
                          "37",
                          "20163"
                        ],
                        "boost": 1
                      }
                    },
                    {
                      "term": {
                        "shipMethod": {
                          "value": "TD_DLV",
                          "boost": 1
                        }
                      }
                    },
                    {
                      "term": {
                        "storeType": {
                          "value": "DS",
                          "boost": 1
                        }
                      }
                    }
                  ],
                  "adjust_pure_negative": true,
                  "boost": 1
                }
              },
              {
                "term": {
                  "category.lcateCd": {
                    "value": 100008,
                    "boost": 1
                  }
                }
              }
            ],
            "adjust_pure_negative": true,
            "boost": 1
          }
        },
        "sort": [
          {
            "adultSort": {
              "order": "asc"
            }
          },
          {
            "itemStoreInfo.weight": {
              "order": "desc"
            }
          },
          {
            "itemStoreInfo.buyerCnt": {
              "order": "desc"
            }
          },
          {
            "itemNo": {
              "order": "desc"
            }
          }
        ]
      }
    '
  ((cnt = "${cnt}" + 1))
done