# LPAR2RRD agent
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# SYNOPSIS
#  lpar2rrd-agent.pl [-s <sample rate>] [-d] [-c] [-n <NMON_DIR> ] [-b <HVMSH_PATH> ] [-i <HVM_IP> ] <LPAR2RRD-SERVER>[:<PORT>]
#
#  -d  forces sending out data immediately to check communication channel (DEBUG purposes)
#  -c  agent collects & sends only internal HMC data
#  -n  agent sends only NMON data from NMON directory <NMON_DIR>
#  -b  path to Hitachi HvmSh API
#  -i  IP adress of HVM (Hitachi Virtualization Manager)
#  -t  <max send time in seconds>
#  -s  <step in seconds>, do not set < 60, do not forget to update crontab line accordingly e.g. -s 300 means in crontab */5 for minutes
#  -m  using sudo for multipath (only root can run it): sudo multipath -l", put this into sudoers: lpar2rrd  ALL = (root) NOPASSWD: /sbin/multipath -ll
#  -x  use Transport Layer Security (TLS)
#
#  options -c and -n are mutual exclusive
#  options -b and -i are both required for Hitachi agent
#  no option - agent collects & sends standard OS agent data
#
#  agent can send data to more servers,
#  each server can have own port to send to
#
#
# OS agent usage: ( http://www.lpar2rrd.com/agent.htm )
# =====================================================
# place me into crontab
# * * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER> > /var/tmp/lpar2rrd-agent.out 2>&1
#
# The agent collects all data and sends them every 10 minutes to LPAR2RRD server
#
# If you use other than standard LPAR2RRD port then place if after SERVER by ':' delimited
# * * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER>:<PORT> > /var/tmp/lpar2rrd-agent.out 2>&1
#
# If you want to send data into more LPAR2RRD server instances (number is not restricted)
# * * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER1>  <LPAR2RRD-SERVER2> > /var/tmp/lpar2rrd-agent.out 2>&1
#
#
# NMON usage: ( http://www.lpar2rrd.com/nmon.htm )
# ================================================
# /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -n <NMON_DIR> <LPAR2RRD-SERVER>  >/var/tmp/lpar2rrd-agent-nmon.out 2>&1
# crontab usage: run it either every 10 minutes to process new data collected by nmon or once a day to get all day in once
# 0,10,20,30,40,50 * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -n <NMON_DIR> <LPAR2RRD-SERVER  > /var/tmp/lpar2rrd-agent-nmon.out 2>&1
#  -n option disables normal agent data collection, only NMON data is collected.
# Use 2 separate crontab lines to get standard OS agent data & NMON data load
#
#
# HMC usage: ( http://www.lpar2rrd.com/hmc_monitoring.htm )
# =========================================================
# . /home/lpar2rrd/lpar2rrd/etc/lpar2rrd.cfg; /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -c <LPAR2RRD-SERVER>  >/var/tmp/lpar2rrd-agent-hmc.out 2>&1
#
# crontab usage: run it every 5 minutes
# 0,5,10,15,20,25,30,35,40,45,50,55 * * * * . /home/lpar2rrd/lpar2rrd/etc/lpar2rrd.cfg; /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -c <LPAR2RRD-SERVER>  > /var/tmp/lpar2rrd-agent-hmc.out 2>&1
# -c option disables normal agent data collection, only internal HMC data is collected.
# Use 2 separate crontab lines to get standard OS agent data & HMC data load
# Notice: when option -c, agent needs some Env info, change the path to etc/lpar2rrd.cfg file according your installation
#
#
# HITACHI usage:
# =========================================================
# place me into crontab
# * * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -b <HVMSH_PATH> -i <HITACHI_HOST_IP> <LPAR2RRD-SERVER> > /var/tmp/lpar2rrd-agent.out 2>&1
#
#
# Use 2 separate crontab lines to get standard OS agent data & HITACHI data load
#
# DEBUG:
# =====
# - option -d forces sending out data immediately to check communication channel
#   /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -d <LPAR2RRD-SERVER>
#
# - error log: /var/tmp/lpar2rrd-agent-*.err
# - data  log: /var/tmp/lpar2rrd-agent-*.txt
# - output log: /var/tmp/lpar2rrd-agent-*.out
#
# Change of frequency sending data to the LPAR2RRD server
# ======================================================
# default behaviour is, that agent tries randomly send data to the LPAR2RRD server between 5 - 20 mins
# you can specify max time when data is send, minimum is 5 minutes
# * * * * * /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl -t <max send time in seconds> <LPAR2RRD-SERVER>
#

#
# protocol 4.0 (all in one line)
# [server*serial]:[lpar name|hmc name]:[lpar id|VMwmare uuid]:time_stamp_unix:time_stamp_text:CPU_frequency|SMT:WPAR_name:WPAR_id:type[O|N|H]:
# [sea|lan|san|mem|cpu|pgs|ame]:name:[IP|WWN|other info]:item1:item2:item3:item4:item5:item6
# type : O - OS agent std (can be even empty)
#        N - NMON
#        H - HMC
#
# Name can be empty for [mem/cpu/pgs/ame]
#
# since 4.70 $version is transferred in ':time_stamp_text version $version:'
#
# data save:
# data/server/hmc/lpar/[lan|san|mem|cpu|pgs|ame|sea]-name.mmm
# data/server/hmc/lpar/[lan|san|mem|cpu|pgs|ame|sea]-name.cfg
#
# To use user defined "df" binary
# * * * * * DF=/usr/local/bin/df /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER> > /var/tmp/lpar2rrd-agent.out 2>&1
#
# How to avoid SAN checks via fcstat (they might cause some problems, it should not happen in v4.50+)
# * * * * * FCSTAT=/bin/true /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER> > /var/tmp/lpar2rrd-agent.out 2>&1
#
# If you prefer to use "svmon -G" instead of "vmstat -v" form memory usage stats, note svmon counts FS cache in different way with different result
# svmon presents FS cache wrong, vmstat does not count large page into pinned ... nice, since 5.05-7 is used vmstat for all except pinned which comes svmon
# * * * * * GET_MEM="svmon -G" /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER>  > /var/tmp/lpar2rrd-agent.out 2>&1
#
# Change sample frequency in crontab, STEP env variable  must be set when it does not run every minute (60 secs)
# 5 minutes (300 seconds)
# 0,5,10,15,20,25,30,35,40,45,50,55 * * * * STEP=300; export STEP; /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER>  > /var/tmp/lpar2rrd-agent.out 2>&1
# 10 minutes (600 secs):
# 0,10,20,30,40,50 * * * * STEP=600; export STEP; /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER>  > /var/tmp/lpar2rrd-agent.out 2>&1
#
# by default only interfaces which have IP adress assiggned are reported, by env variable can this be skiped and selection is done base on LPAR2RRD_LAN_INT env var, it allows regex
# only for Linux, be carefull here to do not stack in 1 graph interfaces from different virtualization level what might lead to creasing of presented traffic by counting some traffic more times
# * * * * * LPAR2RRD_LAN_INT="eth.*0$,bond.*,rhevm,9.*" /usr/bin/perl /opt/lpar2rrd-agent/lpar2rrd-agent.pl <LPAR2RRD-SERVER>  > /var/tmp/lpar2rrd-agent.out 2>&1

# Huawei volume ID need root to get them
# enable sudo for lpar2rrd user, this cmd /usr/sbin/upadm show vlun type=all

my $version = "8.30-4";
print "LPAR2RRD agent version:$version\n" . (localtime) . "\n";
# use Socket;
use IO::Socket::INET;
use strict;
# use Time::Local; # as require in NMON branch

use warnings;
#use File::Copy;  # core module not installed with RHEL9+ perl-base package, own sub used
use Getopt::Std;

use Data::Dumper;
use POSIX qw(mktime strftime);

# sigtrap core module not installed withRHEL9+ perl-base package, %SIG hash used instead
#use sigtrap 'handler', \&my_handler, 'ABRT';
#use sigtrap 'handler', \&my_handler, 'BUS';
#use sigtrap 'handler', \&my_handler, 'SEGV';
#use sigtrap 'handler', \&my_handler, 'SYS';

$SIG{ABRT} = \&my_handler;
$SIG{BUS}  = \&my_handler;
$SIG{SEGV} = \&my_handler;
$SIG{SYS}  = \&my_handler;

# for NMON and HMC possibility is necessary to read it first
my $perl_version = $];    #  5.008008 for 5.8.8
if ( $perl_version < 5.007 ) {

  # our is not supported on perl 5.6
  use vars qw($opt_i $opt_b $opt_n $opt_c $opt_d $opt_t $opt_s $opt_m $opt_w $opt_v $opt_x);
}
else {
  our $opt_n;
  our $opt_c;
  our $opt_d;
  our $opt_b;
  our $opt_i;
  our $opt_t;
  our $opt_s;
  our $opt_m;
  our $opt_w;
  our $opt_v;
  our $opt_x;
}
getopts('vwmdcxn:t:b:i:s:');    # options

# first server name needed for log-, timestamp- and store- file names
if ( $#ARGV < 0 ) {
  print "ERROR You must provide at least one LPAR2RRD server hostname\n";
  exit(1);
}
if ( $opt_c && $opt_n ) {
  print "ERROR both -c and -n options are not possible\n";
  exit(1);
}
if ( ( $opt_b && !$opt_i ) || ( !$opt_b && $opt_i ) ) {
  print "ERROR You must provide both options -b and -i\n";
  exit(1);
}
( my $host, my $port ) = split( ":", $ARGV[0] );
$host =~ s/ .*$//g;
$port =~ s/ .*$//g if defined $port;

if ( $host eq "" ) {
  print "ERROR bad LPAR2RRD server hostname provided\n";
  exit(1);
}

# values are then tested before sending

# Globals
# my $host, my $port
my $port_standard          = "8162";                                                                          # IANA registered port for LPAR2RRD project
my $user_name              = getpwuid($<);
my $base_dir               = "/var/tmp";
my $fs_storae_file         = "$base_dir/lpar2rrd-agent-$host-$user_name-fs.txt";
my $disk_id_tmp_file       = "$base_dir/lpar2rrd-agent-$host-$user_name-diskid.stamp";
my $multipath_tmp_file     = "$base_dir/lpar2rrd-agent-$host-$user_name-multipath.stamp";
my $err_log_tmp_file       = "$base_dir/lpar2rrd-agent-$host-$user_name-errlog.stamp";
my $linux_syslog_tmp_file  = "$base_dir/lpar2rrd-agent-$host-$user_name-dmesg.stamp";
my $linux_syslog_file      = "$base_dir/lpar2rrd-agent-$host-$user_name-last-dmesg.txt";
my $volume_list_tmp_file   = "$base_dir/lpar2rrd-agent-$host-$user_name-lvmlist.stamp";
my $eth_hw_tmp_file        = "$base_dir/lpar2rrd-agent-$host-$user_name-eth-hw.stamp";
my $store_not_confirm_file = "$base_dir/lpar2rrd-agent-$host-$user_name.txt";
my $lastly_created_lines   = "$base_dir/lpar2rrd-agent-$host-$user_name-last.txt";
my $timestamp_file         = "$base_dir/lpar2rrd-agent-$host-$user_name.stamp";
my $timestamp_file_send    = "$base_dir/lpar2rrd-agent-$host-$user_name.stamp-send";                          # UNIX timestamp of latest confirmed saved record on the daemon side
my $trimlogs_run           = "$base_dir/lpar2rrd-agent-$host-$user_name.stamp-trimlogs";
my $RUN_TIME               = 300;                                                                             # send out data every 5 minutes and when random() returns 1
my $MAX_RANDOM_TIME        = 600;                                                                            # maximum time when data is sent to the server (sending is randomized, this is just maximum)
my $protocol_version       = "4.1";
my $error_log              = "$base_dir/lpar2rrd-agent-$host-$user_name.err";
my $act_time_u             = time();
my $act_time               = localtime($act_time_u);
my $last_ok_time           = 0;                                                                               # must be global one due to alarm()
my $first_new_pgs_file     = "$base_dir/lpar2rrd-agent-pgs.cfg";                                              #send paging1 'U' 'U' when creating this file - paging 64kB too
my $iostat_data_file       = "$base_dir/lpar2rrd-agent-iostat-$act_time_u-$$.txt";
my $nfs_iostat_data_file   = "$base_dir/lpar2rrd-agent-nfsiostat-$act_time_u-$$.txt";
my $zfs_iostat_data_file   = "$base_dir/lpar2rrd-agent-$host-zfsiostat-$act_time_u-$$.txt";
my $lsstat_data_file       = "$base_dir/lpar2rrd-agent-lsstat-$act_time_u-$$.txt";
my $wlmstat_data_file      = "$base_dir/lpar2rrd-agent-$host-$user_name-wlmstat-$$.txt";
my $prstat_data_file       = "$base_dir/lpar2rrd-agent-$host-$user_name-prstat-$$.txt";
my $zonestat_data_file     = "$base_dir/lpar2rrd-agent-$host-$user_name-zonestat-$$.txt";
my $poolstat_data_file     = "$base_dir/lpar2rrd-agent-$host-$user_name-poolstat-$$.txt";
my $multi_lin_data_file    = "$base_dir/lpar2rrd-agent-$host-$user_name-multilin-$$.txt";
my $multi_sol_data_file    = "$base_dir/lpar2rrd-agent-$host-$user_name-multisol-$$.txt";
my $multipath_exclude_file = "/opt/lpar2rrd-agent/multipath-exclude.txt";
my $UNAME                  = "uname";
my $type_of_solaris        = `uname -a 2>/dev/null`;
my $os                     = `$UNAME 2>>$error_log`;
my $use_ifconfig           = defined $ENV{LPAR2RRD_USE_IFCONFIG} ? defined $ENV{LPAR2RRD_USE_IFCONFIG} : 0;
my $hvmsh_api_path         = $opt_b;
my $hvmsh_host             = $opt_i;
my $SMBIOS                 = "/usr/sbin/smbios -t SMB_TYPE_SYSTEM";
my $LDMLS                  = "";
my $uptime                 = "uptime";
my $psi_cpu                = "/proc/pressure/cpu";
my $psi_io                 = "/proc/pressure/io";
my $psi_mem                = "/proc/pressure/memory";

if ( -f "/usr/sbin/ldm" ) {
  $LDMLS = "/usr/sbin/ldm ls -p";
}
else {
  $LDMLS = "/sbin/ldm ls -p";
}
my $LDMINFO    = "ldm ls -l -p";
my $LDMLSNET   = "ldm list-netstat -p";
my $LDMLIST    = "ldm list -l -p";
my $ZONEADM    = "/usr/sbin/zoneadm list -vi";
my $ZONEADM_P  = "/usr/sbin/zoneadm list -p";
my $SNEEP      = "sneep";
my $VIRTINFO   = "/usr/sbin/virtinfo -a";
my $SHOWPHYS   = "/usr/sbin/dladm show-phys";
my $SHOWLINK   = "/usr/sbin/dladm show-link";
my $LSDISK     = "ls -l /dev/disk/by-id/wwn* ";
my $LSDISK_AIX = "lsdev -Ccdisk";
my $LSDISK_AIX_HUAWEI = "sudo -n /usr/sbin/upadm show vlun type=all"; # AIX Huawei
my $IOSTAT_SOL = "iostat -En";
my $HOSTID     = "hostid";
my $LSPATH     = "lspath  -F\"name|path_id|connection|parent|path_status|status\"";
my $MULTI_LIN  = "/sbin/multipath -ll";
my $syslog_cmd = "LC_TIME=C dmesg -T --ctime";
# ignore -m, if does not exist /sbin/multipath
if ( $opt_m ) {
  my ($multi_path_bin) = $MULTI_LIN =~ m{^(\S+)};
  unless ( -f $multi_path_bin ) {
    print "WARNING: $multi_path_bin not found, ignoring -m option\n";
    $opt_m = 0;
  }
}
if ( $opt_m && -f "/usr/bin/sudo" ) {
  $MULTI_LIN = "sudo -n /sbin/multipath -ll";
  print "Multipath with sudo: $MULTI_LIN\n";
}
my $MULTI_SOL         = "mpathadm list lu";
my $CPU_LINUX         = "lscpu";
my @list_of_zone_uuid = "";
my @smbios            = "";
my %hmc_conf          = ();
my $aix_cloud         = "lsattr -El sys0 -a partition_uuid";
my $linux_cloud       = "/var/lib/cloud/data/instance-id";
my $azure_skytap      = "/var/lib/skytap/nodeid";
my $list_volumes      = "lspv";
my $lscfg_adapters    = "lscfg";
my $version_aix_error = "1.2";
my $debug_aix_error   = 0;
my $cat_aix_error     = "/bin/cat";
my $inputdir_aix      = $0;
if ( $inputdir_aix  !~ m{^/} ) {
  chomp(my $pwd = `pwd`);
  $inputdir_aix = "$pwd/$inputdir_aix";
  $inputdir_aix =~ s{/[^/]+\.pl(?:/)?}{/};
}else {$inputdir_aix =~ s{/[^/]+\.pl(?:/)?}{/}; }

my %month_map = (
  Jan => 0, Feb => 1, Mar => 2, Apr => 3, May => 4, Jun => 5,
  Jul => 6, Aug => 7, Sep => 8, Oct => 9, Nov => 10, Dec => 11
);

chomp($os);

if ($opt_t) {    # if  -t <sample rate in secs>
  if ( isdigit($opt_t) ) {
    $MAX_RANDOM_TIME = $opt_t;
    print "MAX_RANDOM_TIME has been set in the cmd line, it is changed to: $MAX_RANDOM_TIME\n";
  }
  else {
    print "ERROR: -t parametr was used but no number specified as a parameter: $opt_t, ignoring it and continue\n";
  }
}

# check if we need TLS
my $using_ssl = 0;
if ($opt_x) {
  my $errors  = "";
  my @reqmods = qw(IO::Socket::SSL);

  for my $mod (@reqmods) {
    eval {
      ( my $file = $mod ) =~ s|::|/|g;
      require $file . '.pm';
      $mod->import();
      1;
    } or do {
      $errors .= "$@";
    }
  }

  if ($errors) {
    print "ERROR: -x parametr was used but IO::Socket::SSL module not found, ignoring it and continue unsecured\n";
  }
  else {
    $using_ssl = 1;
  }
}


#
# Trim logs
#
my $trim_max_lines_store = 43000;
my $trim_max_lines_error = 1000 ;
my $trim_file_time_diff = file_time_diff("$trimlogs_run");
if ( !-f "$trimlogs_run" || $trim_file_time_diff > 86400 ) {     # once per day
  trimlog( "$store_not_confirm_file", "$trim_max_lines_store" ); # limit = 43000 lines
  trimlog( "$error_log",              "$trim_max_lines_error" ); # limit =  1000 lines

  # trim nmon logs if exists
  my $agent_nmon_file_for_trim = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.txt";
  my $nmon_error_log_for_trim  = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.err";
  if ( -f "$agent_nmon_file_for_trim" ) {
    trimlog( "$agent_nmon_file_for_trim", "43000" );
  }
  if ( -f "$nmon_error_log_for_trim" ) {
    trimlog( "$nmon_error_log_for_trim", "1000" );
  }

  # trim hitachi logs if exists
  if ($hvmsh_host){
    my $agent_hitachi_file_for_trim = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.txt";
    my $hitachi_error_log_for_trim  = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.err";
    if ( -f "$agent_hitachi_file_for_trim" ) {
      trimlog( "$agent_hitachi_file_for_trim", "43000" );
    }
    if ( -f "$hitachi_error_log_for_trim" ) {
      trimlog( "$hitachi_error_log_for_trim", "1000" );
    } 
  }

  # trim linux and solaris logs if exists
  my $agent_file_for_trim = "$base_dir/lpar2rrd-agent-$host-$user_name.txt";
  my $error_log_for_trim  = "$base_dir/lpar2rrd-agent-$host-$user_name.err";
  if ( -f "$agent_file_for_trim" ) {

    #  trimlog("$agent_file_for_trim", "86400");
  }
  if ( -f "$error_log_for_trim" ) {

    #  trimlog("$error_log_for_trim", "1000");
  }

  my $ret = `touch $trimlogs_run`;    # to keep same routine for all timestamp files
  if ($ret) {
    error( "$ret " . __FILE__ . ":" . __LINE__ );
  }
}

#
# Check if is not running more than 10 agent instances
# If so then exit to do not harm the agent hosted OS
#
my $PROC_RUN_LIMIT = 20;    # to support WPARs where hosted OS see all processes of the WPAR
my @ps             = "";
if ( $os =~ m/SunOS/ ) {

  # include only global processes on the global zone
  my $zonename = `zonename`;
  chomp $zonename;
  if ( $zonename eq "global" ) {
    @ps = grep {/global/} `ps -efZ 2>>$error_log`;
  }
  else {
    @ps = `ps -ef 2>>$error_log`;
  }
}
else {
  my $pscmd = `which ps`;
  chomp $pscmd;
  my $target = readlink($pscmd);
  if ( $target && $target =~ "busybox" ) {
    @ps = `ps w 2>>$error_log`;
  }
  else {
    @ps = `ps -ef 2>>$error_log`;
  }
}

if ( process_limits( $0, $PROC_RUN_LIMIT, \@ps ) == 1 ) {
  error("Exiting as more than $PROC_RUN_LIMIT is running");
  exit 1;
}

error_log_wipe( $error_log, 1024000 );

# enhance PATH environment, ifconfig on Linux can be in /sbin, Solaris and swapping cmd nees /usr/sbin
my $path = $ENV{PATH};
$path .= ":/sbin:/usr/sbin";
$ENV{PATH} = "$path";

# ignore ethernet devices which are closed, AIX 72
$ENV{ENTSTAT_MODE} = "closed.ignore";

# 60 seconds is default step, it could be changed via env variable STEP
my $STEP = 60;
if ( defined $ENV{STEP} && isdigit( $ENV{STEP} ) ) {

  # STEP variable can change step here (vmstat timeout mainly)
  $STEP = $ENV{STEP};
}

# 60 seconds is default step, it can be changed via param -s
if ($opt_s) {
  if ( isdigit($opt_s) ) {
    $STEP = $opt_s;
    print "STEP has been set in the cmd line, it is changed to: $STEP\n";
  }
  else {
    print "ERROR: -s parametr was used but no number specified as a parameter: $opt_s, ignoring it and continue\n";
  }
}

if ( $os =~ m/SunOS/ ) {
  $ENV{LANG} = "C";    # also must be due to parsing of cmd's outputs
}
else {
  $ENV{LANG} = "en_US";    # also must be due to parsing of cmd's outputs
}

if ($opt_d) {              # forces sending out data immediately
  $RUN_TIME = 1;
  $STEP     = 2;           # fast up vmstat --> but also faster time out in alarm!!!
}

#
#-- test if NMON agent
#
my $nmon_data = "";

if ($opt_n) {    # if  -n nmon_data_path
                 # use alert here to be sure that it does not hang
  my $timeout = 36000;    # 10hours, necessary to have here a long timeout to be sure data is loaded for many nmon files!!!
  my $ret     = 0;
  eval {
    local $SIG{ALRM} = sub { die "died in SIG ALRM : agent"; };
    print "Timeout $timeout\n";
    alarm($timeout);

    my $ps_n_count = 0;

    # my @ps         = grep {/$0(?!grep|sh -c)/} `ps -ef 2>>$error_log`; # this is not good
    my @ps = grep { /$0/ && !/sh -c/ } `ps -ef 2>>$error_log`;
    foreach my $ps_line (@ps) {
      print "$ps_line\n";
      if ( $ps_line =~ /\s\-n\s/ && $ps_line =~ /$host/ ) {
        $ps_n_count++;
      }
    }

    #print "\$ps_n_count is $ps_n_count\n";
    if ( $ps_n_count > 1 ) {
      $error_log = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.err";
      error( "Exiting as two NMON agents cant run " . __FILE__ . ":" . __LINE__ );
      exit 1;
    }
    if ( !-d $opt_n ) {
      $error_log = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.err";
      error( "NMON_dir: $opt_n does not exist " . __FILE__ . ":" . __LINE__ );
      exit 1;
    }
    print "NMON_dir set to $opt_n\n";
    $nmon_data = $opt_n;
    $ret       = send_nmon_data();
    alarm(0);
  };

  if ($@) {
    chomp($@);
    if ( $@ =~ /died in SIG ALRM/ ) {
      error( "NMON agent timed out after : $timeout seconds " . __FILE__ . ":" . __LINE__ );
    }
    else {
      error( "NMON agent failed : $@ " . __FILE__ . ":" . __LINE__ );
    }
    exit(1);
  }
  exit($ret);
}

#
# set log names for Hitachi agent
#
if ($opt_b) {
  $store_not_confirm_file = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.txt";
  $timestamp_file         = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.stamp";
  $timestamp_file_send    = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.stamp-send";
  $error_log              = "$base_dir/lpar2rrd-agent-hitachi-$hvmsh_host-$host-$user_name.err";
}

# use alert here to be sure that it does not hang
my $timeout = $STEP * 30;    # necessary to have here a long timeout to be sure data is loaded even after a long outage!!!
my $ret     = 0;
eval {
  local $SIG{ALRM} = sub { die "died in SIG ALRM : agent"; };
  print "main timeout $timeout\n";
  alarm($timeout);
  $ret = main_agent( $act_time_u, $act_time );
  alarm(0);
};

if ($@) {

  # save last OK record time in case of alarm or any other problem (it does not save 0 value)
  keep_time( $last_ok_time, "write" );

  chomp($@);
  if ( $@ =~ /died in SIG ALRM/ ) {
    error( "agent timed out after : $timeout seconds " . __FILE__ . ":" . __LINE__ );
  }
  else {
    if ( $@ =~ /Bad arg length for Socket|Undefined address for Socket/ ) {
      error( "LPAR2RRD server $host was not resolved, try to ping it to confirm DNS works as expect: " . __FILE__ . ":" . __LINE__ );
      #error( "Connection to $host failed, check host resolving: $@ " . __FILE__ . ":" . __LINE__ );
    }
    else {
      error( "agent failed: $@ " . __FILE__ . ":" . __LINE__ );
    }
  }
  if ( -f $iostat_data_file ) {
    unlink($iostat_data_file);    # temporary iostat file, AIX only
  }
  if ( -f $nfs_iostat_data_file ) {
    unlink($nfs_iostat_data_file);
  }
  if ( -f $zfs_iostat_data_file ) {
    unlink($zfs_iostat_data_file);
  }
  if ( -f $zonestat_data_file ) {
    unlink($zonestat_data_file);
  }
  if ( -f $prstat_data_file ) {
    unlink($prstat_data_file);
  }
  if ( -f $wlmstat_data_file ) {
    unlink($wlmstat_data_file);
  }
  if ( -f $poolstat_data_file ) {
    unlink($poolstat_data_file);
  }
  if ( $$ == 1 ) {

    # just to make sure ... would not be good to kill init process
    error( "agent PID is 1!!! " . __FILE__ . ":" . __LINE__ );
    exit(1);
  }
  system("kill -15 $$");    # process group kill
  sleep(1);
  system("kill -9 $$");     # just to be sure
  exit(1);
}

exit($ret);

sub main_agent {
  my $act_time_u = shift;
  my $act_time   = shift;

  # get utilisation
  my $message = '';
  if ($opt_c) {             # -- test if HMC/SDMC agent
    $message = data_collect_hmc( $act_time_u, $act_time );
  }
  elsif ($opt_b) {          # -- test if Hitachi agent
    $message = data_collect_hvmsh( $act_time_u, $act_time );
  }
  else {
    $message = data_collect( $act_time_u, $act_time );
  }

  if ( $message eq '' ) {

    # some problem ...
    return 1;
  }

  store_for_later_transfer( $message, $lastly_created_lines );
  store_for_later_transfer( $message, $store_not_confirm_file );

  # run every $RUN_TIME
  if ( run_now( $act_time_u, $timestamp_file, $RUN_TIME ) == 0 ) {

    # just store data, send them later on
    return 0;
  }

  my $ret = send_data( $store_not_confirm_file, $protocol_version );
  return $ret;
}

sub connect_server {
  my $client           = shift;
  my $protocol_version = shift;

  # contact the server
  if ($using_ssl) {
    if ( open_tls_connection( $client, $host, $port ) == 1 ) {
      return 1;
    }
    else {
      # send version
      $$client->print("$protocol_version\n");
    }
  }
  else {
    if ( open_tcp_connection( $client, $host, $port ) == 1 ) {

      # it does not go here, it crash in above 'eval' and prints its own message
      #error("Error connecting to server at $host:$port: $! ".__FILE__.":".__LINE__);
      # --> no error here, it is already printed in open_TCP
      # in Linux eg.:Bad arg length for Socket::pack_sockaddr_in, length is 0, should be 4 at ...
      return 1;
    }
    else {
      # send version
      $$client->send("$protocol_version\n");
    }
  }
  return 0;
}

sub receive_data {
  my $client                 = shift;
  my $message                = shift;
  my $store_not_confirm_file = shift;

  # receive answer
  my $data_recv;
  if ($using_ssl) {
    chomp ( $data_recv = $$client->readline() );
  }
  else {
    $$client->recv($data_recv, 1024);
    chomp($data_recv);
  }
  my $time_send_orig = "";

  ( my $server, my $lpar, my $lpar_id, my $time_send ) = split( /:/, $message );
  if ( !($lpar_id eq '') && $lpar_id !~ m/-/ && isdigit($lpar_id) && $lpar_id > 1200000000 ) {

    $time_send_orig = $time_send;

    # most probably old data from protocol 1.0 yet in the log
    ( $server, $lpar, $time_send ) = split( /:/, $message );
  }

  #print "-- $data_recv -- $message\n";

  if ( defined($time_send) && isdigit($time_send) && defined($data_recv) && isdigit($data_recv) && ( $time_send == $data_recv || $time_send_orig == $data_recv ) ) {
    return 1;
  }
  else {
    error("wrong server response: agent_time:$time_send $lpar_id $lpar $server: recv_time:$data_recv :");
    return 0;
  }
}

sub open_TCP {

  # get parameters
  my ( $FS, $dest, $port ) = @_;
  my $proto = getprotobyname('tcp');
  socket( $FS, PF_INET, SOCK_STREAM, $proto ) || error( "socket error $dest:$port : $! " . __FILE__ . ":" . __LINE__ ) && return 1;
  my $sin = sockaddr_in( $port, inet_aton($dest) ) || error( "sockaddr_in error $dest:$port : $! " . __FILE__ . ":" . __LINE__ ) && return 1;
  connect( $FS, $sin ) || error( "Connect error $dest:$port : $! " . __FILE__ . ":" . __LINE__ ) && return 1;

  my $old_fh = select($FS);
  $| = 1;    # don't buffer output
  select($old_fh);
  return 0;
}

sub open_tcp_connection {
  my ( $client, $dest, $port ) = @_;
  $$client = IO::Socket::INET->new(
    Type   => SOCK_STREAM,
    Proto    => 'tcp',
    PeerHost => $dest,
    PeerPort => $port)
  || error( "Connect error $dest:$port : $! " . __FILE__ . ":" . __LINE__ ) && return 1;
  $$client->autoflush(1);
  return 0;
}

sub open_tls_connection {
  my ( $client, $dest, $port ) = @_;
  if (open_tcp_connection( $client, $dest, $port ) ) {
    error( "Connect error $dest:$port : $! " . __FILE__ . ":" . __LINE__ ) && return 1;
  }

=pod
  $$client = IO::Socket::SSL->new(
    Domain => AF_INET,
    Type   => SOCK_STREAM,
    Proto    => 'tcp',
    SSL_verify_mode => 0,
    SSL_startHandshake => 0,
    PeerHost => $dest,
    PeerPort => $port,
  ) || error( "Connect error $dest:$port : $!,$SSL_ERROR " . __FILE__ . ":" . __LINE__ ) && return 1;
  $$client->autoflush(1);
=cut

  $$client->print("STARTTLS\n");
  $$client->recv( my $response, 1024 );
  # chomp ( my $response = <$$client> );
  chomp $response;
  print "Daemon response on STARTTLS command: $response\n";
  # upgrade connection to SSL
  if ( $response eq "OK" ) {
    print "Trying to secure existing connection...\n";
    IO::Socket::SSL->start_SSL($$client,
      SSL_verify_mode => 0,
    ) || error( "failed to start SSL on $dest:$port : $!, " . IO::Socket::SSL::errstr() . " " . __FILE__ . ":" . __LINE__ ) && return 1;
    my $ssl_version = eval { $$client->can("get_sslversion") ? $$client->get_sslversion : "n/a" };
    print "new SSL connection with cipher=" . $$client->get_cipher . " version=$ssl_version  certificate:\n".
    "\tsubject=".$$client->peer_certificate('subject')."\n".
    "\tissuer=".$$client->peer_certificate('issuer')."\n";
  }
  else {
    $using_ssl = 0;
    error( "Daemon at $dest:$port doesn't accept STARTTLS command, upgrade it or don't use -x agent parameter" . __FILE__ . ":" . __LINE__ ) && return 1;
  }
  return 0;
}

sub data_collect_hmc {
  my $act_time_u = shift;
  my $act_time   = shift;
  my $DEBUG      = 0;
  my $message    = "";
  my $buffers    = 0;
  $|                      = 1;
  $store_not_confirm_file = "$base_dir/lpar2rrd-agent-hmc-$host-$user_name.txt";
  $timestamp_file         = "$base_dir/lpar2rrd-agent-hmc-$host-$user_name.stamp";
  $timestamp_file_send    = "$base_dir/lpar2rrd-agent-hmc-$host-$user_name.stamp-send";    # UNIX timestamp of latest confirmed saved record on the daemon side
  $error_log              = "$base_dir/lpar2rrd-agent-hmc-$host-$user_name.err";
  my $SSH = $ENV{SSH} . " -q ";                                                            # doubles -q from lpar2rrd.cfg, just to be sure ...
  print "agent HMC for $host\n";

  if ( !defined $ENV{INPUTDIR} ) {
    error("not defined ENV INPUTDIR for HMC/SDMC agent");
    return "";
  }
  my $basedir  = $ENV{INPUTDIR};
  my $wrkdir   = "$basedir/data/";
  my $hmc_list = $ENV{HMC_LIST};
  my @hmc_list_from_json;
  my @hmcs = split( " ", $hmc_list );
  eval { @hmc_list_from_json = getHmcListFromJson(); };
  if ($@) {
    warn( "Something went wrong when getting HMC list in " . __FILE__ . ":" . __LINE__ . "\n" );
  }
  else {
    if (@hmc_list_from_json) {
      @hmcs = @hmc_list_from_json;
    }
  }

  #  if ( !defined $hmcs[0] ) { error("no hmcs for HMC/SDMC agent \$hmc_list is $hmc_list"); return "" }
  if ( !defined $hmcs[0] ) { return "" }    # no error message, it is just filling error log when there is no IBM power monitoring

  print "hmcs for HMC/SDMC agent read:\n", join( "\n", @hmcs ), "\n" if $DEBUG == 3;

  for my $hmc (@hmcs) {
    chomp($hmc);
    my @ivm_list = <$wrkdir/*/$hmc/IVM>;
    my $is_ivm   = 0;
    foreach my $ivm (@ivm_list) {
      $is_ivm = 1;
      last;
    }
    if ( $is_ivm > 0 ) {
      next;    # it is IVM, skip it
    }
    $SSH = $hmc_conf{$hmc}{ssh_key_id} if defined $hmc_conf{$hmc}{ssh_key_id};
    my $USER = $ENV{HMC_USER};
    $USER = $hmc_conf{$hmc}{username} if defined $hmc_conf{$hmc}{username};
    my $ctr_string = "LANG=C; export LANG; $SSH $USER\@$hmc 'LANG=C; export LANG; lshmc -V; cat /proc/meminfo; monhmc -r swap -n 0; cat /proc/stat; exit 2>&1' 2>&1 ";
    my @hmc_info   = `$ctr_string 2>>$error_log`;
    if ( $? == -1 ) {
      my $errno = $!;
      error("agent hmc connection failed: $errno\n @hmc_info\n $ctr_string\n");
      next;
    }
    elsif ( $? != 0 ) {
      my $res = $? >> 8;
      error("agent hmc connection exited with value $res\n @hmc_info\n $ctr_string\n");
      next;
    }

    if ( $message ne "" ) {    # if not first hmc
      $message .= "\n";
    }
    $message .= "hmc-sdmc:$hmc:0:$act_time_u:$act_time\::::H";

    #print "$message\n";

    # cat /proc/meminfo #example
    # MemTotal:      4128368 kB
    # MemFree:        161392 kB
    # Buffers:        397776 kB
    #
    # Cached:        1457064 kB

    my @line_mem = grep {/^MemTotal:/} @hmc_info;
    if ( scalar(@line_mem) ne 1 ) {
      error("bad Mem data-not one line MemTotal for $hmc ,@line_mem,@hmc_info,");
      next;
    }

    #print "mem line is $line_mem[0]\n";
    ( undef, my $size ) = split( ":", $line_mem[0] );
    $size =~ s/\D//g;

    @line_mem = grep {/^MemFree:/} @hmc_info;
    if ( @line_mem ne 1 ) {
      error("bad Mem data-not one line MemFree for $hmc");
      next;
    }

    #print "mem line is $line_mem[0]\n";
    ( undef, my $free ) = split( ":", $line_mem[0] );
    $free =~ s/\D//g;

    @line_mem = grep {/^Cached:/} @hmc_info;
    if ( @line_mem ne 1 ) {
      error("bad Mem data-not one line Cached for $hmc");
      next;
    }

    #print "mem line is $line_mem[0]\n";
    ( undef, my $cached ) = split( ":", $line_mem[0] );
    $cached =~ s/\D//g;

    @line_mem = grep {/^Buffers:/} @hmc_info;
    if ( @line_mem ne 1 ) {
      error("bad Mem data-not one line Cached for $hmc");
      next;
    }

    #print "mem line is $line_mem[0]\n";
    ( undef, $buffers ) = split( ":", $line_mem[0] );
    $buffers =~ s/\D//g;

    $cached = $cached + $buffers;

    my $inuse = $size - $free - $cached;

    #print "\$size $size, \$inuse $inuse, \$free ,$free, \$cached $cached\n";
    $message .= ":mem:::$size:$inuse:$free:$cached:U:U";

    my @line_cpu = grep {/^cpu /} @hmc_info;

    #print "line cpu @line_cpu\n";
    if ( @line_cpu ne 1 ) {
      error("bad cpu data-not one line for $hmc");
      next;
    }

    #print "one line $line_cpu[0]\n";
    ( undef, my $cpu_us, my $cpu_nice, my $cpu_sy, my $cpu_idle, my $cpu_wa ) = split( " ", $line_cpu[0] );

    #  user: normal processes executing in user mode
    #  nice: niced processes executing in user mode
    #  system: processes executing in kernel mode
    #  idle: twiddling thumbs
    #  iowait: waiting for I/O to complete

    $cpu_us += $cpu_nice;

    #print "values % are: $cpu_us, $cpu_sy, $cpu_wa\n";
    $message .= ":cpu:::$cpu_idle:$cpu_sy:$cpu_us:$cpu_wa\::";

    #print "$message\n";

    my $grep_hmc = grep {/SDMC/} @hmc_info;
    if ( $grep_hmc == 1 ) {next}
    ;    # only cpu for SDMC

    $grep_hmc = grep {/HMC/} @hmc_info;
    if ( grep {/HMC/} @hmc_info ne 1 ) {
      error("neither SDMC nor HMC ??? for $hmc");
      next;
    }

    # get paging space usage
    my @line_swap = grep {/Swap:/} @hmc_info;

    #    $line_swap[0] = " Swap:  2040244k total,      220k used,  2040024k free,  1150420k cached";
    #                      Swap:     1992M total,       0Mused,     1992M free,     2432Mcached
    #print "line swap @line_swap\n";
    if ( @line_swap ne 1 ) {
      error("bad Swap data-not one line for $hmc");
      next;
    }
    
    # old method
    #my $split_letter = "k";
    #$split_letter = "M" if ( $line_swap[0] !~ /k/ );
    #( my $paging_space_kb, my $pg_used ) = split( $split_letter, $line_swap[0] );
    #$paging_space_kb =~ s/\D+//;
    #$pg_used         =~ s/\D+//;
    #my $pg_percent = sprintf( "%.1f", $pg_used * 100 / $paging_space_kb );
    #if ( $split_letter eq "M" ) { $paging_space_kb *= 1024; }
    #my $paging_space = sprintf( "%.0f", $paging_space_kb / 1024 );
    
    my ($swap_total, $swap_used, $unit) = (0, 0, 'KiB');

    foreach my $line (@hmc_info) {
      if ($line =~ /Swap:\s+(\d+)\s+total,\s+(\d+)\s+free,\s+(\d+)\s+used\.\s+(\d+)\s+avail\s+Mem/) {
        $swap_total = $1;
        $swap_used  = $3;
        last;
      } elsif ($line =~ /(\w+) Swap:/) {
        $unit = $1;
      }
    }

    if ($unit eq 'MiB') {
      $swap_total *= 1024;  # MiB → KiB
      $swap_used  *= 1024;
    } elsif ($unit eq 'MB') {
      $swap_total *= 1000;  # MB → KiB
      $swap_used  *= 1000;
    } elsif ($unit eq 'GiB') {
      $swap_total *= 1024 * 1024;  # GiB → KiB
      $swap_used  *= 1024 * 1024;
    } elsif ($unit eq 'GB') {
      $swap_total *= 1000 * 1000;  # GB → KiB
      $swap_used  *= 1000 * 1000;
    }

    my $swap_total_mb = $swap_total / 1000;
    my $swap_used_mb  = $swap_used / 1000;   

    my $pg_percent = $swap_total > 0 ? sprintf("%.1f", $swap_used * 100 / $swap_total) : 0;
    my $paging_space = sprintf("%.0f", $swap_total_mb); 
    $message .= ":pgs:::U:U:$paging_space:$pg_percent\::";

    #    print "hmc paging $message\n";
  }
  print "end of hmcs cycle\n";

  #print "$message\n";
  #return "";
  return $message;
}

sub data_collect {
  my $act_time_u = shift;
  my $act_time   = shift;
  my $T0         = time();
  my $T          = $T0;
  my $IOSTAT     = "iostat -Dsal $STEP 1";
  my $IOSTAT_S   = "iostat -Cx $STEP 2";
  my $NFS_IOSTAT = "/usr/sbin/nfsiostat $STEP 2";
  my $WLMSTAT    = "/usr/sbin/wlmstat $STEP 1";
  my $WLMSTAT_M  = "/usr/sbin/wlmstat -M";
  my $WPAR_ID    = "uname -W 2>/dev/null";
  my $DF         = "df";
  if ( defined $ENV{DF} )           { $DF        = $ENV{DF}; } # user "df" binnary setting
  my $DF_AIX     = "$DF -kI";
  my $DF_AIX7    = "$DF -kIT local";                      # AIX 7 supports list of local filesystems only, exclude NFS stuff
  my $DF_SOL     = "$DF -kl";
  my $DF_LIN     = "$DF -klP";
  my $LSDEV_PROC = "lsdev -Cc processor 2>/dev/null";    # SMT purposes
  my $BINDPROC   = "bindprocessor -q 2>/dev/null";       # SMT purposes
  my $AMEPAT     = "amepat";

  # proc0 does not have to exist everywhere!!
  my $CPU_FREQ = "lsattr -El `lscfg -lproc\* 2>>$error_log|cut -d ' ' -f3|head -1` -a frequency -F value 2>/dev/null";    # must be into /dev/null as on ols AIXes does not exist

  my $GET_MEM     = "vmstat -v";                                                                                          # since 4.95-7 is this default
  my $GET_MEM_PIN = "svmon -G";
  my $SYSID       = "lsattr -El sys0";
  my $LPARSTAT    = "lparstat -i";

  #  my $VMSTAT="vmstat -s";
  my $VMSTAT     = "vmstat -sp ALL";                                                                                      # since 4.70
  my $VMSTAT_CPU = "vmstat -I -W -t $STEP 1";                                                                             # for CPU util from OS point of view, Linux and AIX 5.2 use "vmstat $STEP 2"
  if ($opt_w) {
    $VMSTAT_CPU = "vmstat -I -t $STEP 1";
    print "vmstat cpu without -w option: $VMSTAT_CPU\n";
  }
  my $VMSTAT_IOSTAT = "vmstat -d";                                                                                        # since 7.00-4
  my $LIST_DISK     = "lsblk -d";
  my $LSPS          = "lsps -s";

  #my $LSDEV = "lsdev -C -c adapter -F 'name'|grep fcs";
  my $IP_ADDR      = "ip -f inet address";
  my $IP_LINK      = "ip -s link";
  my $IFCONFIG     = "ifconfig -a";
  my $FCSTAT       = "fcstat";                                                                                            # output into /dev/null as on old AIXes does not exist (viz below)
  my $FCSTAT_ALIVE = "lsattr -a attach -El ";                                                                             # to check if the adapter is connected, test only connected adapters
  my $ENTSTAT      = "entstat";
  my $DLSTAT       = "dlstat";
  my $IPADM        = "ipadm";
  my $MEM_MULTIPLY = 4;
  my $LSDEV_C      = "lsdev -C";
  my $UNAMESL      = "uname -sL";
  my $UNAME_N      = "uname -n";
  my $UNAME        = "uname";

  ############ SOLARIS 10 and 11 - ZONE
  #my $PRSTAT_Z = "prstat -Z $STEP 2"; OLD COMMAND
  my $PRSTAT_Z = "prstat -n 30,11 -Z $STEP 2";
  my $ZONESTAT = "/usr/bin/zonestat -r all  -p -T u $STEP 1 2";
  my $POOLSTAT = "poolstat $STEP 2 2>/dev/null";

  my $DEBUG = 3;

  #
  # uptime - duration for which the computer has been powered on
  #
  my $uptime_comm = `$uptime 2>/dev/null`;
  my ($uptime_days) = $uptime_comm =~ /up (.*?),/;

  if ($uptime_days =~ /:/) { # if the uptime is less than one day, it contains a colon
    $uptime_days  =~ s/\s+//g;
    $uptime_days  =~ s/:/h/;
    $uptime_days .= "min";
  }

  #
  # Job TOP start(call sub)
  #

  my $ps_job_output = ps_job();

  # Check if it is not WPAR
  print "$WPAR_ID\n";
  my $wpar_id = `$WPAR_ID`;
  chomp($wpar_id);
  if ( !($wpar_id eq '') && isdigit($wpar_id) && $wpar_id > 0 ) {

    # WPAR must use svmon, vmstat does not provide any data
    #  svmon provides at least per WPAR data: use, pin, FS cache
    $GET_MEM     = "svmon -G";
    $GET_MEM_PIN = "false";      # not for WPAR
    $LSPS        = "false";
  }

  #print "uname\n";
  #my $os =`$UNAME 2>>$error_log`;
  # chomp ($os);

  if ( $os =~ m/SunOS/ ) {
    $UNAMESL     = "false";
    $LSDEV_PROC  = "false";
    $BINDPROC    = "false";
    $CPU_FREQ    = "false";
    $GET_MEM     = "false";
    $GET_MEM_PIN = "false";
    $SYSID       = "false";
    $LPARSTAT    = "false";

    #$VMSTAT = "cat /proc/vmstat";
    $LSPS         = "false";
    $AMEPAT       = "false";
    $LSDEV_C      = "false";
    $IFCONFIG     = "ifconfig -a";
    $FCSTAT       = "false";
    $UNAMESL      = "false";
    $ENTSTAT      = "false";
    $MEM_MULTIPLY = 1;

    $VMSTAT_CPU   = "vmstat $STEP 1";    # for CPU util --> must be 2 as the first data row is the average from the OS start
    $FCSTAT_ALIVE = "false";
  }
  else {
    #print "uname\n";
    #my $os =`$UNAME 2>>$error_log`;
    #chomp ($os);
    if ( $os =~ m/Linux/ ) {
      $LSDEV_PROC = "false";
      $BINDPROC   = "false";
      $CPU_FREQ   = "false";

      #$GET_MEM = "free";
      $GET_MEM     = "cat /proc/meminfo";    # since 4.95-7 is this used
      $GET_MEM_PIN = "false";
      $SYSID       = "false";
      $LPARSTAT    = "false";
      if ( -f "/proc/ppc64/lparcfg" ) {
        $SYSID    = "cat /proc/ppc64/lparcfg";
        $LPARSTAT = "lparstat -i";
      }
      else {
        foreach my $file (</proc/*/lparcfg>) {
          $SYSID = "cat $file";
          last;
        }
      }
      $VMSTAT       = "cat /proc/vmstat";
      $LSPS         = "false";
      $AMEPAT       = "false";
      $LSDEV_C      = "false";
      $IFCONFIG     = "ifconfig -a";
      $FCSTAT       = "false";
      $UNAMESL      = "false";
      $ENTSTAT      = "false";
      $MEM_MULTIPLY = 1;
      $VMSTAT_CPU   = "vmstat $STEP 2";     # for CPU util --> must be 2 as the first data row is the average from the OS start
      $FCSTAT_ALIVE = "false";
    }
  }

  # when FCSTAT=/bin/true in the ENV then avoiding fcstst
  # FCSTAT=/bin/true; export FCSTAT; perl lpar2rrd-agent.pl
  if ( defined $ENV{FCSTAT} )       { $FCSTAT       = $ENV{FCSTAT}; }
  if ( defined $ENV{ENTSTAT} )      { $ENTSTAT      = $ENV{ENTSTAT}; }
  if ( defined $ENV{VMSTAT_CPU} )   { $VMSTAT_CPU   = $ENV{VMSTAT_CPU}; }
  if ( defined $ENV{AMEPAT} )       { $AMEPAT       = $ENV{AMEPAT}; }
  if ( defined $ENV{CPU_FREQ} )     { $CPU_FREQ     = $ENV{CPU_FREQ}; }
  if ( defined $ENV{LSPS} )         { $LSPS         = $ENV{LSPS}; }
  if ( defined $ENV{GET_MEM} )      { $GET_MEM      = $ENV{GET_MEM}; }
  if ( defined $ENV{GET_MEM_PIN} )  { $GET_MEM      = $ENV{GET_MEM_PIN}; }
  if ( defined $ENV{FCSTAT_ALIVE} ) { $FCSTAT_ALIVE = $ENV{FCSTAT_ALIVE}; }
  if ( defined $ENV{SYSID} )        { $SYSID        = $ENV{SYSID}; }

  #my @svmon = `svmon -G -O unit=KB`; --> -O option is supported since AIX 5.3 TL05
  if ( !($wpar_id eq '') && isdigit($wpar_id) && $wpar_id > 0 ) {

    # exclude WPARs
    $CPU_FREQ     = "false";
    $IOSTAT       = "false";
    $VMSTAT       = "vmstat -s ALL";    # since 4.95-4, wpar does not recognizes "-p" option
    $ENTSTAT      = "false";
    $IFCONFIG     = "false";
    $FCSTAT       = "false";
    $FCSTAT_ALIVE = "false";
    $DLSTAT       = "false";
    $LSDEV_C      = "false";
    $BINDPROC     = "false";
    $AMEPAT       = "false";
    $LSDEV_PROC   = "false";
  }

  print "$CPU_FREQ\n";
  my $cpu_freq = "";
  if ( process_limits( "$CPU_FREQ", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$CPU_FREQ\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    $cpu_freq = `$CPU_FREQ 2>>$error_log`;    #
    chomp($cpu_freq);
  }

  print "$LSDEV_PROC\n";
  my @lsdev_proc = "";
  if ( process_limits( "$LSDEV_PROC", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$LSDEV_PROC\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @lsdev_proc = `$LSDEV_PROC 2>>$error_log`;    #
  }

  print "$BINDPROC\n";
  my $bindproc = "";
  if ( process_limits( "$BINDPROC", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$BINDPROC\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    $bindproc = `$BINDPROC 2>>$error_log`;        #
    chomp($bindproc);
  }

  print "$GET_MEM 2>>$error_log\n";
  my @svmon = "";
  if ( process_limits( "$GET_MEM", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$GET_MEM\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @svmon = `$GET_MEM 2>>$error_log`;            # it is in pages
  }

  print "$GET_MEM_PIN 2>>$error_log\n";
  my @svmon_pin = "";
  if ( process_limits( "$GET_MEM_PIN", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$GET_MEM_PIN\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @svmon_pin = `$GET_MEM_PIN 2>>$error_log`;    # it is in pages
  }

  print "$SYSID 2>>$error_log\n";
  my @sysid = "";
  if ( process_limits( "$SYSID", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$SYSID\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @sysid = `$SYSID 2>>$error_log`;              # lsattr -El sys0 -a systemid
  }

  print "$LPARSTAT\n";
  my @lparstat = "";
  if ( process_limits( "$LPARSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$LPARSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @lparstat = `$LPARSTAT 2>/dev/null`;          # must be into /dev/null as on old AIXes does not exist
  }

  print "$VMSTAT 2>>$error_log\n";
  my @vmstat_s = "";
  if ( process_limits( "$VMSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$VMSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @vmstat_s = `$VMSTAT 2>>$error_log`;
  }

  print "$LSPS 2>>$error_log\n";
  my @lsps_s = "";
  if ( process_limits( "$LSPS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$LSPS\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @lsps_s = `$LSPS 2>>$error_log`;
  }

  print "$AMEPAT 2>/dev/null\n";
  my @amepat = "";
  if ( process_limits( "$AMEPAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$AMEPAT\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @amepat = `$AMEPAT 2>/dev/null`;    #svmon -G -O summary=ame
  }

  print "$LSDEV_C 2>>$error_log\n";
  my @lsdev_c = "";
  if ( process_limits( "$LSDEV_C", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$LSDEV_C\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @lsdev_c = `$LSDEV_C 2>>$error_log`;
  }

  my $cloud_uuid = "";

  if ($os =~ m/AIX/) {
    print "$azure_skytap 2>/dev/null\n";

    if (-f $azure_skytap) {
      open( AS, "< $azure_skytap" ) || error( " Can't open $azure_skytap: $! " . __FILE__ . ":" . __LINE__ ) && next;
      $cloud_uuid = <AS>;
      close(AS);
      chomp($cloud_uuid);
      $cloud_uuid =~ s/^vm-//g;
      $cloud_uuid = "skytap_$cloud_uuid";
    } 

    if (!$cloud_uuid) {
      if ( process_limits( "$aix_cloud", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$aix_cloud\", skipping it " . __FILE__ . ":" . __LINE__ );
      } else {
        print "$aix_cloud 2>/dev/null\n";
        $cloud_uuid = `$aix_cloud 2>/dev/null`;
        $cloud_uuid =~ s/partition_uuid\s+//;
        $cloud_uuid =~ s/Partition\s+UUID\s+False//;
        $cloud_uuid =~ s/\s+//;
        chomp($cloud_uuid);
      }
    }

    if (!$cloud_uuid) {
      print "It is not Skytap/IBM Cloud instance\n";
    }
  }

  if ($os =~ m/Linux/){
    if (-f $linux_cloud){
      open( LC, "< $linux_cloud" ) || error( " Can't open $linux_cloud: $! " . __FILE__ . ":" . __LINE__ ) && next;
      $cloud_uuid = <LC>;
      chomp($cloud_uuid);
      close(LC);
    }
  }
  
  my @ifconfig_a = "";
  my $ip_addr;
  my $ip_link;
  if ( $os =~ m/Linux/ && !$use_ifconfig ) {    # on Linuxes try 'ip' command first (if ifconfig is not forced)
    print "$IP_ADDR 2>>$error_log\n";
    print "$IP_LINK 2>>$error_log\n";
    if ( process_limits( $IP_ADDR, $PROC_RUN_LIMIT, \@ps ) ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$IP_ADDR\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    elsif ( process_limits( $IP_LINK, $PROC_RUN_LIMIT, \@ps ) ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$IP_LINK\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      $ip_addr = `$IP_ADDR 2>>$error_log`;
      $ip_link = `$IP_LINK 2>>$error_log`;
    }
  }

  if ( $use_ifconfig || !( defined $ip_addr && defined $ip_link ) ) {    # otherwise use ifconfig
    print "$IFCONFIG 2>>$error_log\n";
    if ( process_limits( "$IFCONFIG", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$IFCONFIG\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @ifconfig_a = `$IFCONFIG 2>>$error_log`;
    }
  }

  my $msglsps = "";                                                      # must be here
  my $lpar_id = "";

  my $model    = "";
  my $systemid = "";
  foreach my $line (@sysid) {
    chomp($line);
    if ( $line =~ m/^modelname/ ) {
      ( my $name, $model ) = split( / +/, $line );
      $model =~ s/IBM,//;
    }
    if ( $line =~ m/^systemid/ ) {
      ( my $name1, $systemid ) = split( / +/, $line );
      $systemid =~ s/IBM,//;
      my $tmp = substr( $systemid, length($systemid) - 7, length($systemid) );    # leave there just last 7 characters what is the serial
      $systemid = $tmp;
    }
    if ( $line =~ m/^system_type=/ ) {

      # Linux
      ( my $name, $model ) = split( /=/, $line );
      if ( $model =~ m/IBM pSeries/ ) {

        # KVM on IBM Power has this in model: system_type=IBM pSeries (emulated by qemu)
        $model = "Linux";
      }
      $model =~ s/IBM,//;
      $model =~ s/ //g;
    }
    if ( $line =~ m/^serial_number=/ ) {

      # Linux on Power
      ( my $name1, $systemid ) = split( /=/, $line );
      $systemid =~ s/IBM,//;
      $systemid =~ s/ //g;
      if ( $systemid !~ m/-/ ) {

        # For IBM Power only but exclude UUID of KVM based VMs :  Serial is : serial_number=829f0112-5699-4350-a48c-4f44e863b19b
        my $tmp = substr( $systemid, length($systemid) - 7, length($systemid) );    # leave there just last 7 characters what is the serial
        $systemid = $tmp;
      }
    }
    if ( $line =~ m/^partition_id=/ ) {

      # Linux
      ( my $name, $lpar_id ) = split( /=/, $line );
      $lpar_id =~ s/ //g;
    }
  }

  if ( $model eq '' ) {
    if ( $os =~ m/Linux/ || $os =~ m/LINUX/ ) {
      $model = "Linux";
    }
    if ( $os =~ m/SunOS/ || $os =~ m/Solaris/ ) {
      $model = "Solaris";
    }
  }

  if ( $model eq '' || $systemid eq '' ) {
    if ( $model !~ m/Linux/ && $model !~ m/Solaris/ ) {
      error( "Model or serial has not been found: model:$model serial:$systemid " . __FILE__ . ":" . __LINE__ );
      return "";
    }
  }
  if ( $model =~ m/NotAvailable/ || $systemid =~ m/NotAvailable/ ) {

    # OS  might sometimes report "NotAvailable" as its serial, it is wrong, skip it
    error( "Model or serial is wrong: model:$model serial:$systemid " . __FILE__ . ":" . __LINE__ );
    return "";
  }
  if ( $model !~ m/-/ && $model !~ m/Linux/ && $model !~ m/Solaris/ ) {

    # wrong model, no "dash" inside, when a problem with RMC then lpar can place there "Machine" string what is wrong!!!
    error( "Model has not been found: model:$model serial:$systemid " . __FILE__ . ":" . __LINE__ );
    return "";
  }

  #
  # find our LPAR name
  #

  my $part_name = "";
  my $node_name = "";    # for wpar name
  foreach my $line (@lparstat) {
    chomp($line);
    if ( $line =~ m/^Partition Name/ ) {
      ( my $name1, $part_name ) = split( /:/, $line );
      $part_name =~ s/^ *//;
    }
    if ( $line =~ m/^Node Name/ ) {
      ( my $name1, $node_name ) = split( /:/, $line );
      $node_name =~ s/^ *//;
    }
  }
  if ( $part_name eq '' ) {

    # AIX 5.1, 5.2 (lparstat does not exist there)
    print "$UNAMESL 2>/dev/null\n";
    $part_name = `$UNAMESL 2>/dev/null`;    # it might fail (returns NULL) on AXI 5.1
    if ( !($part_name eq '') ) {
      chomp($part_name);

      # AIX 2 nim
      ( my $tr1, my $tr2, my $txt ) = split( / /, $part_name );
      $part_name =~ s/^$tr1 $tr2 //;
    }

    if ( $part_name eq "NULL" ) {
      print "$UNAME_N 2>>$error_log\n";
      $part_name = `$UNAME_N 2>>$error_log`;
      chomp($part_name);
    }
  }

  #
  # find our LPAR ID
  #

  foreach my $line (@lparstat) {
    chomp($line);
    if ( $line =~ m/^Partition Number/ ) {
      ( my $name1, $lpar_id ) = split( /:/, $line );
      $lpar_id =~ s/^ *//;
    }
  }

  # linux already use @sysid for lpar_id

  # lpar name for lpars without the HMC control
  my $part_name_tmp = "";
  if ( !($part_name eq '') ) {
    $part_name_tmp = $part_name;
    $part_name_tmp =~ s/-//g;    # AIX places "-" into the partition name's serial
  }
  if ( $part_name_tmp eq '' || $part_name_tmp =~ m/^$systemid$/ ) {

    # when no HMC then systemid is placed into Partition Name
    #   Partition Name                             : 06-28XXX
    #   --> note the slash in the serial!!!
    print "$UNAME_N 2>>$error_log\n";
    $part_name = `$UNAME_N 2>>$error_log`;    #place uname -n as lpar name if nothing works, support lpars without the HMC
    chomp($part_name);
  }

  if ( $part_name eq '' && $lpar_id eq '' ) {
    error( "LPAR name neither LPAR ID have not been found " . __FILE__ . ":" . __LINE__ );
    return "";
  }

  $model     =~ s/:/=====double-colon=====/g;
  $systemid  =~ s/:/=====double-colon=====/g;
  $part_name =~ s/:/=====double-colon=====/g;

  #my $smt = get_smt( $bindproc, \@lsdev_proc );
  my $smt = get_smt( \@amepat );

  my $hw_separator = "*";
  if ( !defined($systemid) || $systemid eq '' ) {
    $hw_separator = "";
  }

  my $uuid_file = "/opt/lpar2rrd-agent/.uuid";

  if ( -f "/etc/vdsm/vdsm.id" ) {

    # KVM hosts (under oVirt)
    $uuid_file = "/etc/vdsm/vdsm.id";
  }

  if ( $lpar_id eq '' && -f $uuid_file ) {

    # Place VMware UUID
    open( FC, "< $uuid_file" ) || error( "Cannot read $uuid_file: $!" . __FILE__ . ":" . __LINE__ );
    my @uuid_all = <FC>;
    close(FC);
    foreach my $line (@uuid_all) {
      chomp($line);
      if ( !($line eq '') && $line =~ m/-/ ) {
        $lpar_id = $line;
        last;
      }
    }
  }
  my @smbios    = "";
  my $uuid_name = "";
  my $grep_s    = `uname -r`;
  chomp $grep_s;
  my $ldom_uuid1   = "";
  my $char_l       = "";
  my $host_id      = "";
  my $role_of_ldom = "";

  if ( $os =~ m/SunOS/ ) {
    if ( $type_of_solaris =~ /sparc/ ) {
      print "$VIRTINFO\n";
      if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        my @ldom_info2 = `$VIRTINFO 2>>$error_log`;
        my ($domain_role) = grep /Domain role:/, @ldom_info2;
        chomp $domain_role;
        $role_of_ldom = $domain_role;
        $role_of_ldom =~ s/Domain role://g;
        print "$domain_role\n";
      }

    }
    my $test_ldom = "";
    if ( $type_of_solaris =~ /sparc/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        if ( $role_of_ldom =~ /LDoms control/ ) {
          print "LDMLS FOUND - $LDMLS:\n";
          $test_ldom = `$LDMLS 2>>$error_log`;
          my $test_ldom_print = $test_ldom;
          chomp $test_ldom_print;
          print "$test_ldom_print\n";
        }
        $host_id = `$HOSTID 2>>$error_log`;
        chomp $host_id;
      }
      $lpar_id = "cdom";
    }
    if ( $test_ldom eq "" ) {
      if ( $type_of_solaris =~ /sparc/ ) {
        my @ldom_info1 = "";
        my $ldom_name  = "";
        print "$VIRTINFO\n";
        if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
        }
        else {
          @ldom_info1 = `$VIRTINFO 2>>$error_log`;
          $host_id    = `$HOSTID 2>>$error_log`;
          chomp $host_id;
          my ($uuid_found) = grep /Domain UUID:/, @ldom_info1;
          chomp $uuid_found;
          ($ldom_name) = grep /Domain name:/, @ldom_info1;
          chomp $ldom_name;
          print "$VIRTINFO - sparc system/$uuid_found\n";
        }
        if ( grep /Domain UUID:/, @ldom_info1 ) {
          ($ldom_uuid1) = grep /Domain UUID:/, @ldom_info1;
          $ldom_uuid1 =~ s/Domain UUID://g;
          $ldom_uuid1 =~ s/\s+//g;
          ($ldom_name) = grep /Domain name:/, @ldom_info1;
          $ldom_name =~ s/Domain name://g;
          $ldom_name =~ s/\s+//g;
          $char_l = "/";
          chomp $ldom_name;
          chomp $ldom_uuid1;
        }
        if ( $ldom_name ne "primary" ) {
          $part_name = "$ldom_name";
          $lpar_id   = "ldom";
          my $zone_exist = `zonename 2>/dev/null`;
          chomp $zone_exist;
          chomp($part_name);
          if ( $grep_s eq "5.10" ) {
            $part_name = `$UNAME_N 2>>$error_log`;
            chomp($part_name);
            if ( -f $LDMLS ) {
              $lpar_id = "ldom";
            }
            else {
              if ( !$ldom_uuid1 ) {
                $lpar_id    = "no_ldom";
                $ldom_uuid1 = "-";
              }
              $lpar_id = "ldom";
              $char_l  = "/";
            }
          }
          if ( $zone_exist ne "global" ) {
            $part_name = `$UNAME_N 2>>$error_log`;
            chomp($part_name);
            $lpar_id = "zone";
            $host_id = `$HOSTID 2>>$error_log`;
            chomp $host_id;
            $ldom_uuid1 = "-";
            $char_l     = "/";
          }
        }
      }
      else {    ################## X86 machines , without SPARC
        if ( process_limits( "$SMBIOS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$SMBIOS\", skipping it " . __FILE__ . ":" . __LINE__ );
        }
        else {
          my $zonename = `zonename`;
          chomp $zonename;
          if ( $zonename eq "global" ) {    ### command smbios not run on zone!!!
            @smbios = `$SMBIOS 2>>$error_log`;
            my ($uuid_found) = grep( /UUID:/, @smbios );
            chomp $uuid_found;
            print "$SMBIOS - not sparc system/$uuid_found\n";
          }
          $host_id = `$HOSTID 2>>$error_log`;
          chomp $host_id;
        }
        ($ldom_uuid1) = grep ( /UUID:/, @smbios );
        $ldom_uuid1 =~ s/UUID://g;
        $ldom_uuid1 =~ s/\s+//g;
        $char_l = "/";
        chomp $ldom_uuid1;
        $lpar_id = "no_ldom";
        my $zone_exist = `zonename 2>/dev/null`;
        chomp $zone_exist;

        if ( $zone_exist ne "global" ) {    ###### zone saved uname -n and host id for daemon
          $part_name = `$UNAME_N 2>>$error_log`;
          chomp($part_name);
          $lpar_id = "zone";
          $host_id = `$HOSTID 2>>$error_log`;
          chomp $host_id;
          $char_l     = "/";
          $ldom_uuid1 = "-";
        }
      }
    }
    else {
      if ( $type_of_solaris =~ /sparc/ ) {
        my @ldom_info1 = "";
        my $ldom_name  = "";
        if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
        }
        else {
          @ldom_info1 = `$VIRTINFO 2>>$error_log`;
          $host_id    = `$HOSTID 2>>$error_log`;
          chomp $host_id;
          my ($uuid_found) = grep /Domain UUID:/, @ldom_info1;
          chomp $uuid_found;
          print "$VIRTINFO - sparc system/$uuid_found\n";
        }
        $lpar_id = "cdom";
        if ( grep /Domain UUID:/, @ldom_info1 ) {
          ($ldom_uuid1) = grep /Domain UUID:/, @ldom_info1;
          $ldom_uuid1 =~ s/Domain UUID://g;
          $ldom_uuid1 =~ s/\s+//g;
          ($ldom_name) = grep /Domain name:/, @ldom_info1;
          $ldom_name =~ s/Domain name://g;
          $ldom_name =~ s/\s+//g;
          $char_l = "/";
          chomp $ldom_uuid1;
        }
        if ( $ldom_name ne "primary" ) {
          $part_name = "$ldom_name";
          $lpar_id   = "ldom";
          my $zone_exist = `zonename 2>/dev/null`;
          chomp $zone_exist;
          if ( $grep_s eq "5.10" ) {
            $part_name = `$UNAME_N 2>>$error_log`;
            chomp($part_name);
            if ( -f $LDMLS ) {
              $lpar_id = "ldom";
            }
            else {
              if ( !$ldom_uuid1 ) {
                $lpar_id    = "no_ldom";
                $ldom_uuid1 = "-";
              }
              $lpar_id = "ldom";
              $char_l  = "/";
            }
          }
          if ( $zone_exist ne "global" ) {    ###### zone saved uname -n and host id for daemon
            $part_name = `$UNAME_N 2>>$error_log`;
            chomp($part_name);
            $lpar_id = "zone";
            $host_id = `$HOSTID 2>>$error_log`;
            chomp $host_id;
            $char_l     = "/";
            $ldom_uuid1 = "-";
          }
        }
      }
      else {                                  ################## X86 machines , without SPARC
        if ( process_limits( "$SMBIOS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$SMBIOS\", skipping it " . __FILE__ . ":" . __LINE__ );
        }
        else {
          my $zonename = `zonename`;
          chomp $zonename;
          if ( $zonename eq "global" ) {      ### command smbios not run on zone!!!
            @smbios = `$SMBIOS 2>>$error_log`;
            my ($uuid_found) = grep( /UUID:/, @smbios );
            chomp $uuid_found;
            print "$SMBIOS - not sparc system/$uuid_found\n";
          }
          $host_id = `$HOSTID 2>>$error_log`;
          chomp $host_id;
        }
        ($ldom_uuid1) = grep ( /UUID:/, @smbios );
        $ldom_uuid1 =~ s/UUID://g;
        $ldom_uuid1 =~ s/\s+//g;
        $char_l = "/";
        chomp $ldom_uuid1;
        $lpar_id = "no_ldom";
        my $zone_exist = `zonename 2>/dev/null`;
        chomp $zone_exist;

        if ( $zone_exist ne "global" ) {
          $part_name = `$UNAME_N 2>>$error_log`;
          chomp($part_name);
          $lpar_id = "zone";
          $host_id = `$HOSTID 2>>$error_log`;
          chomp $host_id;
          $char_l     = "/";
          $ldom_uuid1 = "-";
        }
      }
    }
    if ( $grep_s eq "5.10" ) {
      $model = "Solaris10";
    }
    if ( $grep_s eq "5.11" ) {
      $model = "Solaris11";
    }
  }
  my $message = "$model$hw_separator$systemid:$part_name:$lpar_id$char_l$ldom_uuid1$char_l$host_id:$act_time_u:$act_time version $version:$cpu_freq|$smt:$node_name:$wpar_id\:";

  #  MEM
  #

  my $size            = -1;
  my $inuse           = -1;
  my $free            = -1;
  my $pin             = -1;
  my $virtual         = -1;
  my $available       = -1;
  my $loaned          = 0;
  my $mmode           = -1;
  my $size_pg         = -1;
  my $inuse_pg        = -1;
  my $pin_work        = -1;
  my $pin_pers        = -1;
  my $pin_clnt        = -1;
  my $pin_other       = -1;
  my $in_use_work     = -1;
  my $in_use_pers     = -1;
  my $in_use_clnt     = -1;
  my $page_in         = 0;
  my $page_out        = 0;
  my $paging_space    = -1;
  my $paging_space_kb = "";
  my $pg_free         = "";
  my $pg_used         = "";
  my $shared          = "";
  my $pg_percent      = -1;
  my $loan_exist      = 0;
  my $line_prev       = "";
  my $buffers         = 0;

  #
  #SOLARIS
  # memory
  #
  if ( $os =~ m/SunOS/ ) {
    my $pagesize = `pagesize`;
    ##October 17, 2016 10:33:14 AM CEST
    ##unix:0:system_pages:physmem     520092
    ##zfs:0:arcstats:size     1287883624
    ##unix:0:system_pages:pagesfree   61310

    my @KSTAT_T = `kstat -T d -p :::physmem zfs:::size :::pagesfree 2>>$error_log`;
    my ($physmem) = grep {/physmem/} @KSTAT_T;
    my ( $unix_phy, undef, undef, $c, ) = split( /:/, $physmem );
    chomp($c);
    $c =~ s/physmem//;
    $c =~ s/ //g;
    my $sizes = $c * $pagesize;
    $size = $sizes / 1024;

    #print "SIZE$size\n";
    my ($FS_cache) = grep {/size/} @KSTAT_T;
    my ( $unix_size, undef, undef, $val_a, ) = split( /:/, $FS_cache );
    chomp($val_a);
    $val_a =~ s/size//;
    $val_a =~ s/ //g;
    my $in_use_clnts = $val_a * 1;    # FS cache is not in pages here!
    $in_use_clnt = $in_use_clnts / 1024;

    #print "CLNT$in_use_clnt\n";
    my ($pages_free) = grep {/pagesfree/} @KSTAT_T;
    my ( $free_space, undef, undef, $val_b, ) = split( /:/, $pages_free );
    chomp($val_b);
    $val_b =~ s/pagesfree//;
    $val_b =~ s/ //g;
    my $frees = $val_b * $pagesize;
    $free = $frees / 1024;

    #print "FREE $free\n";
    $inuse = $size - $free;

    #print "INUSE$inuse\n";
  }
  else {

    # vmstat -v all data except pinned which must be from svmon
    # svmon shows FS cache wrong, vmstat pinned
    # WPAR only svmon, it cannot use vmstat

    foreach my $line (@svmon_pin) {
      if ( $line =~ m/^memory/ ) {
        if ( $line =~ m/Shar/ ) {
          ( undef, undef, undef, undef, $pin, undef, undef, undef, undef ) = split( / +/, $line );
        }
        else {
          ( undef, undef, undef, undef, $pin, undef, undef, undef ) = split( / +/, $line );
        }
      }
    }

    foreach my $line (@svmon) {
      chomp($line);
      if ( $GET_MEM =~ m/vmstat -v/ ) {

        # default behaviour since 4.95-7
        $line =~ s/^ *//g;
        if ( $line =~ m/ memory pages/ ) {
          ( $size, my $x ) = split( / +/, $line );
        }
        if ( $line =~ m/ free pages/ ) {
          ($free) = split( / +/, $line );
          if ( isdigit($size) && isdigit($size) && $size > -1 && $free > -1 ) {
            $inuse = $size - $free;
          }
        }
        if ( $line =~ m/ client pages/ ) {
          ($in_use_clnt) = split( / +/, $line );
        }

        # pinned are taken from svmon cmd above @svmon_pin
        #if ( $line =~ m/ pinned pages/ ) {
        #  ($pin) = split( / +/, $line );
        #}

        $line_prev   = $line;
        $in_use_work = 0;
        next;
      }
      if ( $line =~ m/loan/ ) {
        $loan_exist = 1;
      }
      if ( $line =~ m/^memory/ ) {

        # AIX
        if ( $line =~ m/Shar/ ) {
          ( my $name, $size, $inuse, $free, $pin, $virtual, $available, $loaned, $mmode ) = split( / +/, $line );
          if ( $loan_exist == 0 || $loaned eq '' || isdigit($loaned) == 0 ) {

            # svmon -G does not show loaned, svmon -G -O unit=MB does show that!!!
            # get free memory from total - used as a workaround
            $free = $size - $inuse;
          }
          else {
            $free = $free + $loaned;    # AMS support (v4.03), free is free plus loaned memory
          }
        }
        else {
          ( my $name, $size, $inuse, $free, $pin, $virtual, $available, $mmode ) = split( / +/, $line );
          $loaned = 0;
          if ( $available eq '' ) {
            $available = -1;            # AIX 5.1 does not have that
          }
          if ( !defined $mmode || $mmode eq '' ) {
            $mmode = -1;                # AIX 5.1 does not have that
          }
        }
      }

      if ( $line =~ m/^pg space/ ) {
        ( my $name1, my $name2, $size_pg, $inuse_pg ) = split( / +/, $line );
      }
      if ( $line =~ m/^pin/ ) {
        ( my $name, $pin_work, $pin_pers, $pin_clnt, $pin_other ) = split( / +/, $line );
      }
      if ( $line =~ m/^in use/ ) {
        ( my $name1, my $name2, $in_use_work, $in_use_pers, $in_use_clnt ) = split( / +/, $line );
      }

      ###Linux memory###
      if ( $line =~ m/^MemTotal:/ ) {
        ( undef, $size, ) = split( / +/, $line );
      }
      if ( $line =~ m/^MemFree:/ ) {
        ( undef, $free ) = split( / +/, $line );
      }
      if ( $line =~ m/^Shmem:/ ) {
        ( undef, $shared ) = split( / +/, $line );
      }
      if ( $line =~ m/^Cached:/ ) {
        ( undef, $in_use_clnt ) = split( / +/, $line );
        $in_use_clnt = $in_use_clnt + $buffers;

        # Buffers must be added to Cached
      }
      if ( $line =~ m/^MemAvailable:/ ) {
        ( undef, $available ) = split( / +/, $line );
      }
      if ( $line =~ m/^Buffers:/ ) {
        ( undef, $buffers ) = split( / +/, $line );
      }

      $inuse       = $size - $free - $in_use_clnt;
      $in_use_work = $inuse;                         # used mem without the FS cache - FS cache is not here!
      $inuse       = $inuse + $in_use_clnt;
      if ( $line =~ m/^SwapTotal/ ) {
        ( my $name, my $total ) = split( / +/, $line );
        $paging_space_kb = $total;

        #print "$total\n";
      }
      ###Swap###
      if ( $line =~ m/^SwapFree/ ) {
        ( my $name, my $freeswap, ) = split( / +/, $line );

        #print "$freeswap\n";
        $pg_free = $freeswap;
      }
      if ( !($paging_space_kb eq '') && isdigit($paging_space_kb) ) {
        my $pg_used = $paging_space_kb - ( isdigit($pg_free) ? $pg_free : 0 );
        if ( $paging_space_kb == 0 ) {
          $pg_percent   = 100;
        }
        else {
          $pg_percent   = sprintf( "%.1f", $pg_used * 100 / $paging_space_kb );
        }
        $paging_space = $paging_space_kb / 1024;
      }
      $line_prev = $line;
    }
  }

  #if ( $size == 0 ) {
  #   # means svmon failed?? Like AIX 5.1 due to permissions
  #   # use then vmstat -v
  #   # Grrrr : vmstat: Not a recognized flag: v
  #   print "vmstat -v 2>>$error_log\n";
  #   my @vmstat_v = `vmstat -v 2>>$error_log`;
  #   foreach my $line (@vmstat_v) {
  #     chomp ($line);
  #     #if ( $line =~ m/
  #   }
  #}

  if ( $os =~ m/SunOS/ ) {
    my @smbios  = "";
    my $vm_name = `uname -n 2>>/dev/null`;

    #my @list_of_zone_uuid = "";
    if ( process_limits( "$ZONEADM_P", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$ZONEADM_P\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @list_of_zone_uuid = `$ZONEADM_P 2>>$error_log`;
    }
    chomp $vm_name;
    my @ldom_info1 = "";
    if ( $type_of_solaris =~ /sparc/ ) {
      if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        @ldom_info1 = `$VIRTINFO 2>>$error_log`;
      }
    }
    my $ldom_stat = "";
    if ( $role_of_ldom =~ /LDoms control/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        $ldom_stat = `$LDMLS 2>>$error_log`;
      }
    }
    my ( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util, $zone_name, $zone_grep ) = "";
    foreach my $domain ( split( /\n/, $ldom_stat ) ) {
      if ( $domain =~ /DOMAIN|name=/ ) {
        ( undef, $ldom_name, $status, undef, undef, $vcpu, $all_memory, $util, $uptime, $norm_util ) = split( /\|/, $domain );
        $ldom_name  =~ s/name=//g;
        $status     =~ s/state=//g;
        $vcpu       =~ s/ncpu=//g;
        $all_memory =~ s/mem=//g;
        $util       =~ s/util=//g;
        $uptime     =~ s/uptime=//g;
        $norm_util  =~ s/norm_util=//g;
        chomp( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util );
        my @ldom_info2 = `$LDMINFO $ldom_name 2>>$error_log`;
        my ($uuid_ldom) = grep /uuid=/, @ldom_info2;
        $uuid_ldom =~ s/UUID\|uuid=//g;
        chomp $uuid_ldom;
        my @ldomlist = `$LDMLIST $ldom_name 2>>$error_log`;
        my ($ldom_hostname_id) = grep /HOSTID/, @ldomlist;
        $ldom_hostname_id =~ s/HOSTID\|hostid=0x//g;
        chomp $ldom_hostname_id;

        my $old_name = "$ldom_name";    #### primary and secondary ldom_name changed name
        if ( $status ne "active" ) {next}
        my $ldom_string = "_ldom";
        if ( $ldom_name =~ /primary/ ) {
          my $host_name = `hostname`;
          chomp $host_name;
          $ldom_name = "$host_name";
        }
        if ( $ldom_name =~ /secondary/ ) {
          my $host_name = `hostname`;
          chomp $host_name;
          $host_name =~ s/-primary//g;
          $host_name =~ s/-control//g;
          $ldom_name = "$host_name-secondary";
        }
        $message .= ":$ldom_name$ldom_string:$uuid_ldom$char_l$ldom_hostname_id:$status:$vcpu:$all_memory:$util:$uptime:$norm_util\:";
      }
    }
  }

  if ( $os =~ m/SunOS/ ) {
    my $zonename = `zonename`;
    chomp $zonename;
    if ( $zonename eq "global" ) {
      $message .= ":mem:::$size:$inuse:$free:0:0:$in_use_clnt";
    }
    if ( $grep_s eq "5.8" ) { # Solaris 8
      $message .= ":mem:::$size:$inuse:$free:0:0:$in_use_clnt";
    }
  }
  else {
    # everything is in pages, to kB multiply by 4
    $size        *= $MEM_MULTIPLY;
    $inuse       *= $MEM_MULTIPLY;
    $free        *= $MEM_MULTIPLY;
    $pin         *= $MEM_MULTIPLY;
    $in_use_work *= $MEM_MULTIPLY;
    $in_use_clnt *= $MEM_MULTIPLY;
    $message .= ":mem:::$size:$inuse:$free:$pin:$in_use_work:$in_use_clnt";
  }

  #
  # SOLARIS
  #
  #
  # Paging 1
  #
  my (@match_pg) = grep {/pages swapped /} @vmstat_s;
  my ( $match_solaris, $val_a, $val_b, $c ) = split( / +/, @match_pg );
  if ( @match_pg == 2 ) {

    #Solaris
    my $SWAP_SOL     = "swap -l";
    my @swap_solaris = "";
    if ( process_limits( "$SWAP_SOL", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$SWAP_SOL\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @swap_solaris = `$SWAP_SOL 2>>$error_log`;
    }
    my $pg_total_count;
    my $pg_free_total;
    foreach my $line (@swap_solaris) {
      if ( $line =~ m/^\/dev/ ) {

        # Solaris
        # get paging space usage already here for Solaris
        #            total       used       free
        #Swap:      1015800      39772     976028
        my ( $solaris, undef, undef, $pg_total, $pg_free ) = split( / +/, $line );
        $pg_total       *= 0.512;
        $pg_free        *= 0.512;
        $pg_total_count += $pg_total;
        $pg_free_total  += $pg_free;
      }
    }
    my $paging_space_kb = $pg_total_count;
    my $pg_used_total   = $pg_total_count - $pg_free_total;
    if ( !($paging_space_kb eq '') && isdigit($paging_space_kb) ) {
      $pg_percent   = sprintf( "%.1f", $pg_used_total * 100 / $paging_space_kb );
      $paging_space = sprintf( "%d",   $paging_space_kb / 1024 );
    }
    $page_in = $match_pg[0];
    chomp($page_in);
    $page_in =~ s/pages swapped in//;    # Solaris
    $page_in =~ s/ //g;
    @match_pg = grep {/pages swapped out/} @vmstat_s;
    $page_out = $match_pg[0];
    chomp($page_out);
    $page_out =~ s/pages swapped out/ /;    # Solaris
    $page_out =~ s/ //g;
  }
  else {
    @match_pg = grep {/pswpin /} @vmstat_s;
    if ( @match_pg == 1 ) {                 # Linux
      $page_in = $match_pg[0];
      chomp($page_in);
      $page_in =~ s/pswpin //;              # Linux
      $page_in =~ s/ //g;
      @match_pg = grep {/pswpout /} @vmstat_s;
      $page_out = $match_pg[0];
      chomp($page_out);
      $page_out =~ s/pswpout //;            # Linux
      $page_out =~ s/ //g;
    }
    else {
      #AIX
      @match_pg = grep {/paging space page ins/} @vmstat_s;
      if ( @match_pg == 1 || @match_pg == 2 ) {    # 4k only
        $page_in = $match_pg[0];
        chomp($page_in);
        $page_in =~ s/paging space page ins//;
        $page_in =~ s/ //g;
        @match_pg = grep {/paging space page outs/} @vmstat_s;
        $page_out = $match_pg[0];
        chomp($page_out);
        $page_out =~ s/paging space page outs//;
        $page_out =~ s/ //g;
      }
      else {
        if ( @match_pg == 3 ) {                    # 4k and 64k too !
                                                   # 4k workout
          $page_in = $match_pg[1];                 # 4k workout
          chomp($page_in);
          $page_in =~ s/paging space page ins//;
          $page_in =~ s/ //g;
          my $page_in_64 = $match_pg[2];
          @match_pg = grep {/paging space page outs/} @vmstat_s;
          $page_out = $match_pg[1];
          chomp($page_out);
          $page_out =~ s/paging space page outs//;
          $page_out =~ s/ //g;

          # 64k workout
          chomp($page_in_64);
          $page_in_64 =~ s/paging space page ins//;
          $page_in_64 =~ s/ //g;
          $page_in += $page_in_64 * 16;
          my $page_out_64 = $match_pg[2];
          chomp($page_out_64);
          $page_out_64 =~ s/paging space page outs//;
          $page_out_64 =~ s/ //g;
          $page_out += $page_out_64 * 16;
        }
        else {
          print "could not resolve Paging 1\n";
        }
      }
    }
  }

  #  foreach my $line (@vmstat_s) {
  #    chomp ($line);
  #    if ( $line =~ m/paging space page ins/ || $line =~ m/^pswpin / ) {
  #      $page_in = $line;
  #      $page_in =~ s/paging space page ins//;
  #      $page_in =~ s/pswpin //; # Linux
  #      $page_in =~ s/ //g;
  #    }
  #    if ( $line =~ m/paging space page outs/ || $line =~ m/^pswpout /) {
  #      $page_out = $line;
  #      $page_out =~ s/paging space page outs//;
  #      $page_out =~ s/pswpout //; # Linux
  #      $page_out =~ s/ //g;
  #    }
  #  }

  #
  # Paging 2
  #

  while ( my $line = shift @lsps_s ) {

    # AIX, linux already done in memory part above
    #print "$line\n";
    chomp($line);
    if ( ( $line =~ m/Total/ ) && ( $line =~ m/Paging/ ) && ( $line =~ m/Space/ ) ) {
      $line = shift @lsps_s;

      #print "$line\n";
      chomp($line);
      ( undef, $paging_space, $pg_percent ) = split( / +/, $line );    # MegaBytes
                                                                       #print "$paging_space, $pg_percent\n";
      $paging_space =~ s/MB//;
      $pg_percent   =~ s/\%//;
      last;
    }
  }
  if ( $os =~ m/AIX/ ) {
    if ( !-f $first_new_pgs_file ) {
      my $ret = `touch $first_new_pgs_file`;                           # to keep same routine for all timestamp files
      if ($ret) {
        error( "$ret " . __FILE__ . ":" . __LINE__ );
      }
      $page_in  = 'U';
      $page_out = 'U';
    }
  }

  $message .= ":pgs:::$page_in:$page_out:$paging_space:$pg_percent\::";

  #
  # SEA (only VIOS/AIX of course)
  # it print stats only for real adapters (backed device)
  #

  foreach my $line (@lsdev_c) {
    chomp($line);
    if ( $line =~ m/^ent/ && $line =~ m/Available/ && $line =~ m/Shared Ethernet Adapter/ ) {
      ( my $en, undef ) = split( / +/, $line );
      if ( process_limits( "$ENTSTAT -d $en", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$ENTSTAT -d $en \", skipping it " . __FILE__ . ":" . __LINE__ );
        last;
      }
      print "ENTSTAT_MODE=1 $ENTSTAT -d $en 2>/dev/null\n";    # must be to /dev/null, otherwise it prints empty rows to the error file
                                                               # ENTSTAT_MODE=1 --> ignore not used sea devices, otherwise en error in errpt can be printed
                                                               # The ENTSTAT_MODE variable is added by apar IV50358 (AIX 6.1 TL09 SP06 and higher), IV69207 (AIX 7.1 TL04 SP00 and higher), VIOS 2.2.3.60 and higher.
      my @entstat_d = `ENTSTAT_MODE=1 $ENTSTAT -d $en 2>/dev/null`;
      my $back_en   = "";
      my $found     = 0;
      my $pct_trans = "";
      my $pct_rec   = "";

      foreach my $ent_line (@entstat_d) {
        chomp($ent_line);
        if ( $ent_line =~ m/Real Adapter:/ ) {
          $found   = 1;
          $back_en = $ent_line;
          $back_en =~ s/^.*ent/ent/;
          next;
        }
        if ( $found == 1 && $ent_line =~ m/^Packets:/ ) {
          $ent_line =~ s/Packets://g;
          $ent_line =~ s/^ *//;
          ( $pct_trans, $pct_rec ) = split( / +/, $ent_line );
          $pct_trans =~ s/ //g;
          $pct_rec   =~ s/ //g;
        }
        if ( $found == 1 && $ent_line =~ m/^Bytes:/ ) {
          $ent_line =~ s/Bytes://g;
          $ent_line =~ s/^ *//;
          ( my $transb, my $recb ) = split( / +/, $ent_line );
          $transb  =~ s/ //g;
          $recb    =~ s/ //g;
          $en      =~ s/:/-/g;    # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing, same problem with IPv6 like fe80::5cb7:2ff:f....
          $back_en =~ s/:/-/g;    # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing, same problem with IPv6 like fe80::5cb7:2ff:f....
          $message .= ":sea:$back_en:$en:$transb:$recb:$pct_trans:$pct_rec\::";
          last;
        }
      }
    }
  }

  #
  # LAN
  #

  # by default only interfaces which have IP adress assiggned are reported, by env variable can this be skiped and selection is done base on LPAR2RRD_LAN_INT env var, it allows regex
  # only for Linux
  # export LPAR2RRD_LAN_INT="eth.*,bond.*,rhevm,9.*"
  my $interface_selection = "";
  if ( defined $ENV{LPAR2RRD_LAN_INT} ) { $interface_selection = $ENV{LPAR2RRD_LAN_INT}; }
  chomp($interface_selection);

  # by default 'ip' command is used for obtaining lan data on Linux, this can be changed by LPAR2RRD_USE_IFCONFIG env var set to 1
  # export LPAR2RRD_USE_IFCONFIG=1
  if ( !$use_ifconfig && defined $ip_addr && defined $ip_link ) {    # ip command
    my %devices;
    my @interfaces = split /^\d+: /m, $ip_addr;                      #split to get individual interfaces
    shift @interfaces;
    $interface_selection =~ s/,/|/g;

    for my $interface (@interfaces) {
      my @int_arr = split "\n", $interface;                          # split to get each line
      my $first   = 0;
      my $name    = "";
      my $inet    = "";
      for my $int (@int_arr) {
        if ( $first == 0 ) {
          $first++;
          $name = $int;
          $name =~ s/: .*$//g;
        }
        if ( $int =~ /\s+inet\s(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*/ ) {
          ($inet) = ($1);
          if ( !( $name eq "" ) && !( $name eq "lo" ) && ( !$interface_selection || $name =~ /$interface_selection/ ) ) {
            $name =~ s/@.*//; # must be filtered "@bond0" from bond0.1542@bond0 as this is not a real name of the interface
            $devices{$name} = $inet;
          }
          $name = "";
          $inet = "";
        }
      }
    }

    @interfaces = split /^\d+: /m, $ip_link;    #split to get individual interfaces
    shift @interfaces;
    for my $interface (@interfaces) {
      if ( $interface =~ /([^:@]+).*\n.*\n\s+RX:.*\n\s+(\d+)\s+(\d+).*\n\s+TX:.*\n\s+(\d+)\s+(\d+).*\n/ ) {
        my ( $name, $RXB, $RXP, $TXB, $TXP ) = ( $1, $2, $3, $4, $5 );
        my $inet = $devices{$name};

        # skip Docker related eth interface, not related ones
        if ( $name =~ m/^br-|^docker|^veth/ ) {
          next;
        }
        my $mac_info =  "cat /sys/class/net/$name/address";
        my $mac_address = "";
        if (! -f $mac_info ) {
          if ( process_limits( "$mac_info", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$mac_info\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            $mac_address = `$mac_info 2>/dev/null`;
            if ($mac_address) {
              chomp $mac_address;
              $mac_address =~ s/:/=====double-colon=====/g;
            }
          }
        }
        if ( $interface_selection && $name =~ /$interface_selection/ ) {
          # it is reporter because it passes LPAR2RRD_LAN_INT="^enp1,^et" ...
          $message .= ":lan:$name:$inet:$TXB:$RXB:$TXP:$RXP:$mac_address\:";
        }
        else {
          if ( defined $inet ) {
            $message .= ":lan:$name:$inet:$TXB:$RXB:$TXP:$RXP:$mac_address\:";
          }
        }
      }
    }
  }    ## if (defined $ip_addr && defined $ip_link && !$use_ifconfig)
  else {    # ifconfig command
    my $en   = "";
    my $inet = "";
    my $rx7  = 0;
    my $tx7  = 0;
    my $rp7  = 0;
    my $tp7  = 0;
    my $pass = 1;

    # we select only interfaces which have set IP adress
    #
    #
    #lo0: flags=2001000849<UP,LOOPBACK,RUNNING,MULTICAST,IPv4,VIRTUAL> mtu 8232 index 1
    #inet 127.0.0.1 netmask ff000000
    #net0: flags=100001000843<UP,BROADCAST,RUNNING,MULTICAST,IPv4,PHYSRUNNING> mtu 1500 index 2
    #inet 192.168.188.18 netmask ffffff00 broadcast 192.168.188.255

    while ( my $line = shift @ifconfig_a ) {
      chomp($line);
      if ( $os =~ m/SunOS/ && $grep_s eq "5.11" ) {
        ##  Solaris
        print "$IPADM 2>/dev/null\n";
        my @ipadm_list = `$IPADM 2>/dev/null`;
        if ( $type_of_solaris =~ /sparc/ ) { next; }
        ( my $en, undef ) = split( /:/, $line );

        #next if ( $line !~ m/^net/ && $line !~ m/^vnet/ );
        next if ( !defined($en) || $en eq '' || $en !~ m/^[a-z]/ );

        $line = shift @ifconfig_a;
        next if ( $line =~ m/inet6/ );

        chomp($line);
        ( my $inett, my $inet ) = split( / +/, $line );
        $inet =~ s/:/./g;
        if ( $inett == m/inet/ ) {

          #LINK    IPKTS   RBYTES    OPKTS   OBYTES
          #net0  1260971  173163224   762220  145879618
          my ($grep_net) = grep ( /$en/, @ipadm_list );
          if ($grep_net) {
            my ( undef, $stat, $status ) = split( /\s+/, $grep_net );
            next if ( $stat ne "ip" );
            next if ( $status ne "ok" );
          }
          print "grep {/net/} `$DLSTAT -u R $en 2>/dev/null\n`";
          my @entstat_d = grep {/net/} `$DLSTAT -u R $en 2>/dev/null`;
          if ( !($entstat_d[0] eq '') ) {
            chomp( $entstat_d[0] );
            my ( $entstat_solaris, undef, $recp, $recb, $transp, $transb ) = split( / +/, $entstat_d[0] );
            if ( $recp ne '' && $recb ne '' && $transp ne '' && $transb ne '' ) {
              $en   =~ s/:/-/g;    # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing
              $inet =~ s/:/-/g;    # replace : by "-" due to IPV6 adresses
              $message .= ":lan:$en:$inet:$transb:$recb:$transp:$recp\::";
            }
            $inet = "";
            $en   = "";
            next;
          }
        }
      }
      elsif ( $os =~ m/SunOS/ && $grep_s eq "5.10" ) {
        print "  no dlstat data, probably Sol10\n";
      }

      if ( $os =~ m/AIX/ ) {
        ( my $en, undef ) = split( /:/, $line );
        if ( $line =~ m/^en/ ) {
          $line = shift @ifconfig_a;
          chomp($line);
          ( my $inett, my $inet ) = split( / +/, $line );
          if ( $inett =~ m/inet/ ) {
            if ( process_limits( "$ENTSTAT -d $en", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
              error( "there is already running $PROC_RUN_LIMIT copies of \"$ENTSTAT -d $en \", skipping it " . __FILE__ . ":" . __LINE__ );
              last;
            }
            print "grep {/Bytes:|^Packets:/} \`$ENTSTAT -d $en 2>/dev/null\'\n";    # must be to /dev/null, otherwise it prints empty rows to the error file
            my @entstat_d = grep {/Bytes:|^Packets:/} `$ENTSTAT -d $en 2>/dev/null`;
            if ( !($entstat_d[0] eq '') && $entstat_d[0] =~ /Packets/ && !($entstat_d[1] eq '') && $entstat_d[1] =~ /Bytes/ ) {
              chomp( $entstat_d[0] );
              ( undef, my $transp, undef, my $recp ) = split( / +/, $entstat_d[0] );

              #print "en $en inet $inet transp $transp recp $recp\n";
              chomp( $entstat_d[1] );
              ( undef, my $transb, undef, my $recb ) = split( / +/, $entstat_d[1] );

              my @entstat_d_test      = `$ENTSTAT -d $en 2>/dev/null`;
              my ($trans_errors)      = "";
              my ($rec_errors)        = "";
              my ($packets_dropped_t) = "";
              my ($packets_dropped_r) = "";
              my ($bad_packets)       = "";
              if ( ( grep( /Transmit Errors:/, @entstat_d_test ) ) ) {
                ($trans_errors)       = grep /Transmit Errors:/, @entstat_d_test;
                if ($trans_errors){
                  $trans_errors    =~ s/Transmit Errors://g;
                  $trans_errors    =~ s/^\s+//g;
                  $trans_errors   =~ s/Receive Errors: \d+$//g;
                  $trans_errors    =~ s/\s+$//g;
                  if ( ! isdigit($trans_errors) ) {
                    $trans_errors = 0;
                  }
                  else{
                    if ($trans_errors < 0){
                      $trans_errors = 0;
                    }
                  }
                }
              }
              else { $trans_errors = 0 }
              if ( ( grep( /Receive Errors:/, @entstat_d_test ) ) ) {
                ($rec_errors)       = grep /Receive Errors:/, @entstat_d_test;
                if ($rec_errors){
                  $rec_errors    =~ s/Transmit Errors: \d//g;
                  $rec_errors    =~ s/Receive Errors://g;
                  $rec_errors    =~ s/\s//g;
                  if ( ! isdigit($rec_errors) ) {
                    $rec_errors = 0;
                  }
                  else{
                    if ($rec_errors < 0){
                      $rec_errors = 0;
                    }
                  }
                }
              }
              else { $rec_errors = 0 }
              if ( ( grep( /Packets Dropped:/, @entstat_d_test ) ) ) {
                ($packets_dropped_t)       = grep /Packets Dropped:/, @entstat_d_test;
                if ($packets_dropped_t){
                  $packets_dropped_t    =~ s/Packets Dropped: \d$//g;
                  $packets_dropped_t    =~ s/^Packets Dropped://g;
                  $packets_dropped_t    =~ s/\s//g;
                  if ( ! isdigit($packets_dropped_t) ) {
                    $packets_dropped_t = 0;
                  }
                  else{
                    if ($packets_dropped_t < 0){
                      $packets_dropped_t = 0;
                    }
                  }
                }
              }
              else { $packets_dropped_t = 0 }
              if ( ( grep( /Packets Dropped:/, @entstat_d_test ) ) ) {
                ($packets_dropped_r)       = grep /Packets Dropped:/, @entstat_d_test;
                if ($packets_dropped_r){
                  $packets_dropped_r    =~ s/^Packets Dropped: \d//g;
                  $packets_dropped_r    =~ s/\s//g;
                  $packets_dropped_r    =~ s/PacketsDropped://g;
                  if ( ! isdigit($packets_dropped_r) ) {
                    $packets_dropped_r = 0;
                  }
                  else{
                    if ($packets_dropped_r < 0){
                      $packets_dropped_r = 0;
                    }
                  }
                }
              }
              else { $packets_dropped_r = 0 }
              if ( ( grep( /Bad Packets:/, @entstat_d_test ) ) ) {
                ($bad_packets)       = grep /Bad Packets:/, @entstat_d_test;
                if ($bad_packets){
                  $bad_packets    =~ s/\s//g;
                  $bad_packets    =~ s/BadPackets://g;
                  if ( ! isdigit($bad_packets) ) {
                    $bad_packets = 0;
                  }
                  else{
                    if ($bad_packets < 0){
                      $bad_packets = 0;
                    }
                  }
                }
              }
              else { $bad_packets = 0 }
              my $lan_error_total = "";
              if ( $bad_packets ne '' && !($packets_dropped_t ne '') && $packets_dropped_r ne '' && $rec_errors ne '' && $trans_errors ne '' ) {
                $lan_error_total = $bad_packets+$packets_dropped_t+$packets_dropped_r+$rec_errors+$trans_errors;
              }
              #print "en $en inet $inet transb $transb recb $recb\n";
              if ( !($recb eq '') && !($transb eq '') && !($recp eq '') && !($transp eq '') ) {
                $en   =~ s/:/-/g;                                                   # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing, same problem with IPv6 like fe80::5cb7:2ff:f....
                $inet =~ s/:/-/g;                                                   # replace : by "-" du to IPV6 adresses
                my $mac_address = "";
                if ( ( grep( /Hardware Address:/, @entstat_d_test ) ) ) {
                  ($mac_address) = grep /Hardware Address:/, @entstat_d_test;
                  $mac_address        =~ s/Hardware Address://g;
                  $mac_address        =~ s/:/=====double-colon=====/g;
                  $mac_address        =~ s/\s+//g;
                }
                $message .= ":lan:$en:$inet:$transb:$recb:$transp:$recp:$mac_address\:";
                if ($lan_error_total ne ""){
                  $message .= ":lan_error:$en:$inet:$lan_error_total:\::::";
                }
                $inet = "";
                $en   = "";
                next;
              }
            }
          }
        }
      }

      if ( $os =~ m/Linux/ ) {
        if ( $line eq '' || $line =~ m/^lo/ ) {
          $en   = "";
          $inet = "";
          $pass = 1;
          $tp7  = 0;
          $rp7  = 0;
          next;
        }
        if ( $line =~ m/^[a-z,A-Z,0-9]/ ) {
          ( $en, undef ) = split( / +/, $line );
          my $ensave = $en;
          my $chr    = chop($ensave);
          if ( $chr eq ':' ) {
            $en = $ensave;
          }
          $inet = "";
          $pass = 1;
          $tp7  = 0;
          $rp7  = 0;
          next;
        }
        if ( $en eq '' ) {
          $inet = "";
          $pass = 1;
          $tp7  = 0;
          $rp7  = 0;
          next;
        }
        if ( $inet eq '' && $pass == 1 ) {
          if ( !($interface_selection eq '') ) {

            # check if $en pass interface check (LPAR2RRD_LAN_INT env var)
            my @condition_all = split( /,/, $interface_selection );
            $pass = 0;
            foreach my $cond (@condition_all) {
              chomp($cond);
              if ( $en =~ m/$cond/ ) {
                $pass = 1;
                $tp7  = 0;
                $rp7  = 0;
                last;
              }
            }
            if ( $pass == 0 ) {
              next;
            }
          }
        }
        if ( $line =~ m/inet addr:/ ) {

          # inet addr:12.34.56.78  Bcast:12.34.56.255  Mask:255.255.255.0
          ( my $trash, my $inett, $inet ) = split( / +/, $line );
          $inet =~ s/^addr://;
          next;
        }
        if ( $line =~ m/inet\s+[0-9]/ ) {

          # inet 12.34.56.78  netmask 255.255.255.0  broadcast 12.34.56.255
          ( my $trash, my $inett, $inet ) = split( / +/, $line );
          next;
        }

        if ( $line =~ m/RX packets:/ && !($en eq '') && $pass == 1 ) {

          # RHEL 6
          $rp7 = $line;
          $rp7 =~ s/^.*RX packets://;
          $rp7 =~ s/ .*$//;
          if ( isdigit($rp7) == 0 ) { $rp7 = 0; }
          next;
        }
        if ( $line =~ m/TX packets:/ && !($en eq '') && $pass == 1 ) {

          # RHEL 6
          $tp7 = $line;
          $tp7 =~ s/^.*TX packets://;
          $tp7 =~ s/ .*$//;
          if ( isdigit($tp7) == 0 ) { $tp7 = 0; }
          next;
        }

        if ( $line =~ m/RX bytes:/ && !($en eq '') && $pass == 1 ) {
          if ( !($interface_selection eq '') || !($inet eq '') ) {

            # if not defined $inet the nthe interface does not have defined IPv4 --> it is probably used for bonding or for other purposes and stats are available on elevel above
            # if is defined $interface_selection (LPAR2RRD_LAN_INT) then consider even interfaces without IP
            # if not defined $inet the nthe interface does not have defined IPv4 --> it is probably used for bonding or for other purposes and stats are available on elevel above
            # RX bytes:2807052391 (2.6 GiB)  TX bytes:561573504 (535.5 MiB)

            # RHEL 6
            # RX packets:33928217 errors:0 dropped:0 overruns:0 frame:0
            # TX packets:18312543 errors:0 dropped:0 overruns:0 carrier:0
            # RX bytes:5214787166 (4.8 GiB)  TX bytes:3173288350 (2.9 GiB)

            # RHEL 7
            # RX packets 112532720  bytes 14153747142 (13.1 GiB)
            # TX packets 48944232  bytes 5085864223 (4.7 GiB)

            my $rx     = 0;
            my $recb   = 0;
            my $transb = 0;
            foreach my $item ( split( / /, $line ) ) {
              if ( $item =~ m/bytes:/ && $rx == 0 ) {
                $recb = $item;
                $recb =~ s/bytes://;
                $rx++;
                next;
              }
              if ( $item =~ m/bytes:/ && $rx == 1 ) {
                $transb = $item;
                $transb =~ s/bytes://;
                if ( !($recb eq '') && !($transb eq '') ) {
                  my @ifconfig_mac      = `$IFCONFIG $en 2>/dev/null`;
                  my $mac_address = "";
                  if ( ( grep( /ether/, @ifconfig_mac  ) ) ) {
                    my ($mac_address_line) = grep /ether/, @ifconfig_mac;
                    if ($mac_address_line =~ /([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})/) {
                      $mac_address = $1;
                    }
                  }
                  $en   =~ s/:/-/g;    # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing
                  $inet =~ s/:/-/g;    # replace : by "-" du to IPV6 adresses
                  if ( $tp7 == 0 ) { $tp7 = ""; }
                  if ( $rp7 == 0 ) { $rp7 = ""; }
                  $message .= ":lan:$en:$inet:$transb:$recb:$tp7:$rp7:$mac_address\:";
                  last;
                }
                $rx++;
              }
            }
            $en   = "";                # null eth name
            $inet = "";
            next;
          }
        }
        if ( $line =~ m/RX packets / && !($en eq '') && $pass == 1 ) {
          if ( !($interface_selection eq '') || !($inet eq '') ) {

            # if not defined $inet the nthe interface does not have defined IPv4 --> it is probbaly used for bonding or for other purposes and stats are available on elevel above
            # if is defined $interface_selection (LPAR2RRD_LAN_INT) then consider even interfaces without IP
            # RX packets 4597169  bytes 2622810752 (2.4 GiB)
            ( undef, undef, undef, $rp7, undef, $rx7 ) = split( / +/, $line );
            next;
          }
        }
        if ( $line =~ m/TX packets / && !($en eq '') && $pass == 1 ) {
          if ( !($interface_selection eq '') || !($inet eq '') ) {
            my @ifconfig_mac      = `$IFCONFIG $en 2>/dev/null`;
            my $mac_address = "";
            if ( ( grep( /ether/, @ifconfig_mac  ) ) ) {
              my ($mac_address_line) = grep /ether/, @ifconfig_mac;
              if ($mac_address_line =~ /([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})/) {
                $mac_address = $1;
              }
            }
            # if not defined $inet the nthe interface does not have defined IPv4 --> it is probbaly used for bonding or for other purposes and stats are available on elevel above
            # if is defined $interface_selection (LPAR2RRD_LAN_INT) then consider even interfaces without IP
            # TX packets 4597169  bytes 2622810752 (2.4 GiB)
            ( undef, undef, undef, $tp7, undef, $tx7 ) = split( / +/, $line );
            $en   =~ s/:/-/g;    # replace : by "-" --> eth0:2 --> eth0-2 as ":" is a problem for parsing
            $inet =~ s/:/-/g;    # replace : by "-" du to IPV6 adresses
            $message .= ":lan:$en:$inet:$tx7:$rx7:$tp7:$rp7:$mac_address\:";
            $rx7  = 0;
            $tx7  = 0;
            $rp7  = 0;
            $tp7  = 0;
            $en   = "";          # null eth name
            $inet = "";
            $pass = 1;
          }
        }
      }
    }
  }    ## else [ if (defined $ip_addr && defined $ip_link && !$use_ifconfig)]

  #
  # AME
  #

  while ( my $line = shift @amepat ) {

    #print "$line\n";
    chomp($line);
    if ( $line =~ m/^Compressed/ ) {
      ( undef, undef, undef, my $comem ) = split( / +/, $line );    # MegaBytes
      $comem = -1 if ( $comem eq "N/A" );
      $line  = shift @amepat;
      chomp($line);
      $line = shift @amepat if ( $line eq "" );
      chomp($line);
      my $coratio = -1;
      if ( $line =~ m/^Compression/ ) {
        ( undef, undef, $coratio ) = split( / +/, $line );
        if ( $coratio eq "N/A" ) {
          $coratio = -1;
        }
        else {
          if ( isdigit($coratio) ) {
            $coratio = 100 * $coratio;    # tenth and hundredth are needed
          }
          else {
            $coratio = -1;
          }
        }
      }
      my $codefic = 0;                    # use 0 here, Deficit does not have to be listed in amepat when there is no real memory deficit
      $line = shift @amepat;
      if ( !defined $line ) {

        # amepat sometimes ends here, probably when there is no deficit
        chomp($comem);
        chomp($coratio);
        chomp($codefic);                  # just to be sure
        $message .= ":ame:::$comem:$coratio:$codefic\:::";
        last;
      }
      chomp($line);
      $line = shift @amepat if ( $line eq "" );
      if ( !defined $line ) {

        # amepat sometimes ends here, probably when there is no deficit
        chomp($comem);
        chomp($coratio);
        chomp($codefic);                  # just to be sure
        $message .= ":ame:::$comem:$coratio:$codefic\:::";
        last;
      }
      chomp($line);
      if ( $line =~ m/^Deficit/ ) {
        ( undef, undef, undef, undef, $codefic ) = split( / +/, $line );
        $codefic = -1 if ( $codefic eq "N/A" );
      }

      #$msgame=":ame_mem:$comem:$coratio:$codefic";
      chomp($comem);
      chomp($coratio);
      chomp($codefic);    # just to be sure
      $message .= ":ame:::$comem:$coratio:$codefic\:::";
      last;
    }
  }

  #
  # fork iostat
  #

  my $pid_iostat;
  if ( $os =~ /AIX|SunOS/ ) {
    $pid_iostat = fork();
    if ( !defined $pid_iostat ) {
      die "Cannot fork: $!";
    }
    elsif ( $pid_iostat == 0 ) {
      print "fork iostat 2>>$error_log\n";

      eval {
        local $SIG{ALRM} = sub { die "died in SIG ALRM : agent"; };

        # it uses the same timeout as the main
        $timeout = $timeout - 30;    # to kill at first fork and then main, if main kill fork process at first then iostat will survive
        print "iostat fork timeout $timeout\n";
        alarm($timeout);
        if ( $os eq "SunOS" ) {
          iostat( $iostat_data_file, $IOSTAT_S );
        }
        else {
          iostat( $iostat_data_file, $IOSTAT );
        }
        alarm(0);
      };

      if ($@) {
        chomp($@);
        if ( $@ =~ /died in SIG ALRM/ ) {
          error( "iostat fork timed out after : $timeout seconds " . __FILE__ . ":" . __LINE__ );
          if ( $$ == 1 ) {

            # just to make sure ... would not be good to kill init process
            error( "agent fork PID is 1!!! " . __FILE__ . ":" . __LINE__ );
            exit(1);
          }
          system("kill -15 $$");    # process group kill
          sleep(1);
          system("kill -9 $$");     # just to be sure
        }
      }
      exit(0);
    }
  }

  #
  # fork NFS iostat
  #
  my $pid_nfs_iostat;
  if ( $os =~ /Linux/ ) {
    my $NFS_check = "/usr/sbin/nfsiostat";
    if ( ! -x $NFS_check ) {
      print "nfsiostat is not available, it is part of nfs-utils (RHEL) or nfs-common (Debian)\n";
    } else {
      $pid_nfs_iostat = fork();
      if ( !defined $pid_nfs_iostat ) {
        die "Cannot fork: $!";
      }
      elsif ( $pid_nfs_iostat == 0 ) {
        print "fork nfs iostat 2>>$error_log\n";

        # no need alarm here, it is used for nfsiostat cmd in nfs_iostat()
        nfs_iostat( $nfs_iostat_data_file, $NFS_IOSTAT );
        exit(0);
      }
    }
  }

  #
  # fork ZFS iostat
  #
  my $pid_zfs_iostat;
  if ( $os =~ /Linux/ ) {
    my $zpool_bin = `which zpool 2>/dev/null`;
    chomp $zpool_bin;

    if (!$zpool_bin) {
    print "ZFS is not installed (cmd 'zpool' not found)\n";
    } else {
      $pid_zfs_iostat = fork();
      if ( !defined $pid_zfs_iostat ) {
        die "Cannot fork: $!";
      }
      elsif ( $pid_zfs_iostat == 0 ) {
        print "fork zfs iostat 2>>$error_log\n";

        eval {
          local $SIG{ALRM} = sub { die "died in SIG ALRM : agent"; };

          # it uses the same timeout as the main
          $timeout = $timeout - 30;    # to kill at first fork and then main, if main kill fork process at first then iostat will survive
          print "zfs iostat fork timeout $timeout\n";
          alarm($timeout);
          zfs_iostat( $zfs_iostat_data_file);
          alarm(0);
        };

        if ($@) {
          chomp($@);
          if ( $@ =~ /died in SIG ALRM/ ) {
            error( "zfs iostat fork timed out after : $timeout seconds " . __FILE__ . ":" . __LINE__ );
            if ( $$ == 1 ) {

              # just to make sure ... would not be good to kill init process
              error( "agent fork PID is 1!!! " . __FILE__ . ":" . __LINE__ );
              exit(1);
            }
            system("kill -15 $$");    # process group kill
            sleep(1);
            system("kill -9 $$");     # just to be sure
          }
        }
        exit(0);
      }
    }
  }

  #
  # fork WLM
  #

  my $pid_wlmstat = 0;
  if ( $os =~ /AIX/ && ( $wpar_id eq '' || !($wpar_id > 0) || !isdigit($wpar_id) ) ) {
    $pid_wlmstat = fork();
    if ( !defined $pid_wlmstat ) {
      die "Cannot fork: $!";
    }
    elsif ( $pid_wlmstat == 0 ) {
      print "Fork wlmstat 2>>$error_log\n";
      eval {
        local $SIG{ALRM} = sub { die "died in SIG ALRM: agent"; };
        $timeout = $timeout - 30;
        print "wlmstat fork timeout $timeout\n";
        alarm($timeout);
        wlmstat( $wlmstat_data_file, $WLMSTAT, $WLMSTAT_M );
        alarm(0);
      };
      if ($@) {
        chomp($@);
        if ( $@ =~ /died in SIG ALRM/ ) {
          error( "Wlmstat fork timed out after : $timeout second " . __FILE__ . ":" . __LINE__ );
          if ( $$ == 1 ) {
            error( "agent fork PID is 1!!! " . __FILE__ . ":" . __FILE__ );
            exit(1);
          }    ##end if ( $$ == 1)
          system("kill -15 $$");
          sleep(1);
          system("kill -9 $$");
        }
      }
      exit(0);
    }
  }

  #
  #
  # VMSTAT IOSTAT/DATA
  my $time_before_iostat;
  $time_before_iostat = time();
  my @iostat_data_60sec_before;
  if ( $os =~ /Linux/ ) {
    my @data_all;
    my @all_disk;
    print "$VMSTAT_IOSTAT 2>>$error_log\n";

    my $vmstat_iostat_cmd = `$VMSTAT_IOSTAT`;
    # LIST_DISK - command "lsblk -d"
    #NAME MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
    #sda    8:0    0   30G  0 disk
    #sdb    8:16   0  100G  0 disk
    #sdc    8:32   0  100G  0 disk
    #sdd    8:48   0  100G  0 disk
    #sde    8:64   0  100G  0 disk
    my $list_disk_cmd     = `$LIST_DISK`;

    @data_all = split( "\n", $vmstat_iostat_cmd );
    @all_disk = split( "\n", $list_disk_cmd );
    ########
    ## only discs filtered
    ########
    my @list_all_disk;
    foreach my $disk (@all_disk) {
      if ( $disk =~ /^NAME/ ) {next}
      my ($disk_name) = grep /^[hsv]d[a-z] |^[hsv]d[a-z][a-z] |^nvme\dn\d |^nvme\dn\d\d |^xvd[a-z] |^xvd[a-z][a-z] /, $disk;    # the better way getting list of disk, because of older Linux systems
       # old ATA (hda, hdb, etc.) and NVME SSD (nvme0n1, etc.) disks added [JD]
       # added vd* xvd* -PH : Xen paravirt drivers use /dev/xvda (first detected disk) and the standard Linux virtio drivers use /dev/vda (first detected disk) for example.

      #my ($disk_name,undef,undef,undef,undef,$type) = split( " ", $disk );
      #if ($type ne "disk"){next}
      if ( defined $disk_name ) {
      push @list_all_disk, $disk_name;
      }

      #print "$disk_name,$type\n";
    }

    # example vmstat -d
    #disk-  ----------------------reads---------------------- ----------------------writes------------------     ---------IO----------
    #         total        merged       sectors       ms         total      merged      sectors        ms           cur    sec
    #sda    1634283       1006175      104625925    23516511   11067778   151168038    1344712666   256232139       0     11174
    #sdb    287721229        0         8378313040   2120485344 119393504      0        6916278528   448543354       0     2205250
    #sdc    8815480          0         282670596    60559759   4594035     130016      173197104    11009287        0     48199
    # ( NO DISK NEXT ) dm-3   2963479    0      94832698 21128205 1588454      0 52877088 11407542    0  22303
    # ( NO DISK NEXT ) dm-4   1453426    0      46510954 10914786 857065      0 29324416 7068943      0  11774
    foreach my $line (@data_all) {

      #print "$line\n";
      my ( $disk_name, $iops_read, undef, $data_read, $ms_read, $iops_write, undef, $data_write, $ms_write ) = split( " ", $line );
      my $if_disk = grep {/$disk_name/} @list_all_disk;
      if ( $if_disk != 1 ) {next}
      my $line_data = ":vmstat_io:$disk_name:$iops_read:$data_read:$ms_read:$iops_write:$data_write:$ms_write\n";
      push @iostat_data_60sec_before, $line_data;
    }
  }

  #
  #
  # fork SOLARIS10 prstat
  my $pid_prstat = 0;
  if ( $os =~ /SunOS/ && $grep_s eq "5.10" ) {
    $pid_prstat = fork();
    if ( !defined $pid_prstat ) {
      die "Cannot fork: $!";
    }
    elsif ( $pid_prstat == 0 ) {
      print "Fork prstat 2>>$error_log\n";
      eval {
        local $SIG{ALRM} = sub { die "died in SIG ALRM: agent"; };

        $timeout = $timeout - 30;
        print "prstat fork timeout $timeout\n";
        alarm($timeout);
        prstat( $prstat_data_file, $PRSTAT_Z );
        alarm(0);
      };

      if ($@) {
        chomp($@);
        if ( $@ =~ /died in SIG ALRM/ ) {
          error( "Prstat fork timed out after : $timeout second " . __FILE__ . ":" . __LINE__ );
          if ( $$ == 1 ) {
            error( "agent fork PID is 1!!! " . __FILE__ . ":" . __FILE__ );
            exit(1);
          }    ##end if ( $$ == 1)
          system("kill -15 $$");
          sleep(1);
          system("kill -9 $$");
        }
      }
      exit(0);
    }
  }

  #
  #
  # fork SOLARIS11 zonestat
  my $pid_zonestat = 0;
  if ( $os =~ /SunOS/ && $grep_s eq "5.11" ) {
    $pid_zonestat = fork();
    if ( !defined $pid_zonestat ) {
      die "Cannot fork: $!";
    }
    elsif ( $pid_zonestat == 0 ) {
      print "Fork zonestat 2>>$error_log\n";
      eval {
        local $SIG{ALRM} = sub { die "died in SIG ALRM: agent"; };

        $timeout = $timeout - 30;
        print "zonestat fork timeout $timeout\n";
        alarm($timeout);
        zonestat( $zonestat_data_file, $ZONESTAT );
        alarm(0);
      };

      if ($@) {
        chomp($@);
        if ( $@ =~ /died in SIG ALRM/ ) {
          error( "Zonestat fork timed out after : $timeout second " . __FILE__ . ":" . __LINE__ );
          if ( $$ == 1 ) {
            error( "agent fork PID is 1!!! " . __FILE__ . ":" . __FILE__ );
            exit(1);
          }    ##end if ( $$ == 1)
          system("kill -15 $$");
          sleep(1);
          system("kill -9 $$");
        }
      }
      exit(0);
    }
  }

  #
  #
  # fork SOLARIS11 poolstat
  my $pid_poolstat = 0;
  if ( $os =~ /SunOS/ && $grep_s =~ /5\.10|5\.11/ ) {
    $pid_poolstat = fork();
    if ( !defined $pid_poolstat ) {
      die "Cannot fork: $!";
    }
    elsif ( $pid_poolstat == 0 ) {
      print "Fork poolstat 2>>$error_log\n";
      eval {
        local $SIG{ALRM} = sub { die "died in SIG ALRM: agent"; };

        $timeout = $timeout - 30;
        print "poolstat fork timeout $timeout\n";
        alarm($timeout);
        poolstat( $poolstat_data_file, $POOLSTAT );
        alarm(0);
      };

      if ($@) {
        chomp($@);
        if ( $@ =~ /died in SIG ALRM/ ) {
          error( "Poolstat fork timed out after : $timeout second " . __FILE__ . ":" . __LINE__ );
          if ( $$ == 1 ) {
            error( "agent fork PID is 1!!! " . __FILE__ . ":" . __FILE__ );
            exit(1);
          }    ##end if ( $$ == 1)
          system("kill -15 $$");
          sleep(1);
          system("kill -9 $$");
        }
      }
      exit(0);
    }
  }

  #
  # CPU util for dedicated partitions
  # $entitled can be used for storing and graphing "pc" vmstat for WPARs only!
  #

  my $dedicated   = 0;
  my $entitled    = 0;
  my $lpars_lines = 0;
  foreach my $line (@lparstat) {
    chomp($line);
    $lpars_lines++;
    if ( $line =~ m/^Type/ && $line =~ m/Dedicated/ ) {
      $dedicated = 1;
    }
    if ( $dedicated == 1 && $line =~ m/^Entitled Capacity/ ) {
      ( my $name1, $entitled ) = split( /:/, $line );
      $entitled =~ s/^ *//;
      $entitled =~ s/ //;
      $entitled =~ s/\.00//;
      last;
    }
  }

  ######################################################################## SOLARIS|NETSTAT - first collecting data phys-addapters
  my ( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util ) = "";
  my $timestamp_before = time();
  my %hash_net         = ();
  if ( $os =~ m/SunOS/ ) {
    my $ldom_stat = "";
    if ( $role_of_ldom =~ /LDoms control/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        $ldom_stat = `$LDMLS 2>>$error_log`;
      }
    }
    foreach my $domain ( split( /\n/, $ldom_stat ) ) {
      if ( $domain =~ /DOMAIN|name=/ ) {
        if ( $role_of_ldom =~ /LDoms control/ ) {
          ( undef, $ldom_name, $status, undef, undef, $vcpu, $all_memory, $util, $uptime, $norm_util ) = split( /\|/, $domain );
          $ldom_name =~ s/name=//g;
          chomp $ldom_name;
          print "$LDMLSNET from $ldom_name\n";
          my $net_stat = "";
          if ( process_limits( "$LDMLSNET", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLSNET\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            $net_stat = `$LDMLSNET $ldom_name`;
            print "$net_stat\n";
          }
          print "$LDMINFO\n";
          my @ldom_info3 = "";
          if ( process_limits( "$LDMINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            @ldom_info3 = `$LDMINFO $ldom_name 2>>$error_log`;
          }
          print "$SHOWPHYS\n";
          my @ldom_phys = "";
          if ( process_limits( "$SHOWPHYS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$SHOWPHYS\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            @ldom_phys = `$SHOWPHYS 2>>$error_log`;
          }
          my ($uuid_ldom) = grep /uuid=/, @ldom_info3;
          $uuid_ldom =~ s/UUID\|uuid=//g;
          chomp $uuid_ldom;
          foreach my $net_ldom ( split /\n/, $net_stat ) {
            if ( $net_ldom =~ /link/ ) {
              my ( $link_name, $ipackets, $rbytes, $opackets, $obytes ) = split( /\|/, $net_ldom );
              $link_name =~ s/link=//g;
              $ipackets  =~ s/ipackets=//g;
              $rbytes    =~ s/rbytes=//g;
              $opackets  =~ s/opackets=//g;
              $obytes    =~ s/obytes=//g;
              my ($grep_phys) = grep /$link_name/, @ldom_phys;
              if ($grep_phys) {
                my ( undef, undef, $status ) = split( /\s+/, $grep_phys );
                if ( $status eq "up" ) {

                  if ( $ipackets =~ /M$/ ) {
                    $ipackets =~ s/M//g;
                    $ipackets = $ipackets * 1000;
                  }
                  elsif ( $ipackets =~ /G$/ ) {
                    $ipackets =~ s/G//g;
                    $ipackets = $ipackets * 1000 * 1000;
                  }
                  if ( $opackets =~ /M$/ ) {
                    $opackets =~ s/M//g;
                    $opackets = $opackets * 1000;
                  }
                  elsif ( $opackets =~ /G$/ ) {
                    $opackets =~ s/G//g;
                    $opackets = $opackets * 1000 * 1000;
                  }
                  if ( $rbytes =~ /M$/ ) {
                    $rbytes =~ s/M//g;
                    $rbytes = $rbytes * 1000;
                  }
                  elsif ( $rbytes =~ /G$/ ) {
                    $rbytes =~ s/G//g;
                    $rbytes = $rbytes * 1000 * 1000;
                  }
                  if ( $obytes =~ /M$/ ) {
                    $obytes =~ s/M//g;
                    $obytes = $obytes * 1000;
                  }
                  elsif ( $obytes =~ /G$/ ) {
                    $obytes =~ s/G//g;
                    $obytes = $obytes * 1000 * 1000;
                  }
                  $ipackets =~ s/K|R//g;
                  $opackets =~ s/K|R//g;
                  $rbytes   =~ s/K|R//g;
                  $obytes   =~ s/K|R//g;
                  $hash_net{$uuid_ldom}{$link_name}{values} = "$ipackets,$rbytes,$opackets,$obytes";
                }
              }
            }
          }
        }
      }
    }
  }

  ######################################################################## SOLARIS|NETSTAT - first collecting data logical-adappters / only no-control ldoms
  my ( $ldom_name1, $status1, $vcpu1, $all_memory1, $util1, $uptime1, $norm_util1 ) = "";
  my $timestamp_before1 = time();
  my %hash_net1         = ();
  if ( $os =~ m/SunOS/ ) {
    my $ldom_stat1 = "";
    if ( $role_of_ldom =~ /LDoms control/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        $ldom_stat1 = `$LDMLS 2>>$error_log`;
      }
    }
    foreach my $domain ( split( /\n/, $ldom_stat1 ) ) {
      if ( $role_of_ldom =~ /LDoms control/ ) {
        if ( $domain =~ /DOMAIN|name=/ ) {
          ( undef, $ldom_name1, $status1, undef, undef, $vcpu1, $all_memory1, $util1, $uptime1, $norm_util1 ) = split( /\|/, $domain );
          $ldom_name1 =~ s/name=//g;
          print "$LDMLSNET no-control ldom $ldom_name1\n";
          my $net_stat1 = "";
          if ( process_limits( "$LDMLSNET", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLSNET\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            $net_stat1 = `$LDMLSNET $ldom_name1`;
          }
          my @ldom_info4 = "";
          if ( process_limits( "$LDMINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            @ldom_info4 = `$LDMINFO $ldom_name1 2>>$error_log`;
          }
          print "$SHOWLINK from no-control ldom $ldom_name1\n";
          my @log_adap = "";
          if ( process_limits( "$SHOWLINK", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$SHOWLINK\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            @log_adap = `$SHOWLINK 2>>$error_log`;
          }
          my @ldom_info1 = "";
          if ( $type_of_solaris =~ /sparc/ ) {
            if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
              error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
            }
            else {
              @ldom_info1 = `$VIRTINFO 2>>$error_log`;
            }
          }
          my ($uuid_ldom) = grep /uuid=/, @ldom_info4;
          $uuid_ldom =~ s/UUID\|uuid=//g;
          chomp $uuid_ldom;
          my ($ldom_role) = grep /Domain name/, @ldom_info1;
          foreach my $net_ldom ( split /\n/, $net_stat1 ) {
            if ( $net_ldom =~ /link/ ) {
              my ( $link_name, $ipackets, $rbytes, $opackets, $obytes ) = split( /\|/, $net_ldom );
              $link_name =~ s/link=//g;
              $ipackets  =~ s/ipackets=//g;
              $rbytes    =~ s/rbytes=//g;
              $opackets  =~ s/opackets=//g;
              $obytes    =~ s/obytes=//g;
              my ($grep_log) = grep /^$link_name/, @log_adap;
              if ($grep_log) {
                my ( $name, $eth, $speed, $status ) = split( /\s+/, $grep_log );
                if ( $eth eq "phys" ) { next; }
                if ( $ipackets =~ /M$/ ) {
                  $ipackets =~ s/M//g;
                  $ipackets = $ipackets * 1000;
                }
                elsif ( $ipackets =~ /G$/ ) {
                  $ipackets =~ s/G//g;
                  $ipackets = $ipackets * 1000 * 1000;
                }
                if ( $opackets =~ /M$/ ) {
                  $opackets =~ s/M//g;
                  $opackets = $opackets * 1000;
                }
                elsif ( $opackets =~ /G$/ ) {
                  $opackets =~ s/G//g;
                  $opackets = $opackets * 1000 * 1000;
                }
                if ( $rbytes =~ /M$/ ) {
                  $rbytes =~ s/M//g;
                  $rbytes = $rbytes * 1000;
                }
                elsif ( $rbytes =~ /G$/ ) {
                  $rbytes =~ s/G//g;
                  $rbytes = $rbytes * 1000 * 1000;
                }
                if ( $obytes =~ /M$/ ) {
                  $obytes =~ s/M//g;
                  $obytes = $obytes * 1000;
                }
                elsif ( $obytes =~ /G$/ ) {
                  $obytes =~ s/G//g;
                  $obytes = $obytes * 1000 * 1000;
                }
                $ipackets =~ s/K|R//g;
                $opackets =~ s/K|R//g;
                $rbytes   =~ s/K|R//g;
                $obytes   =~ s/K|R//g;
                $hash_net1{$uuid_ldom}{$link_name}{values} = "$ipackets,$rbytes,$opackets,$obytes";
              }
            }
          }
        }
      }
    }
  }

  #print Dumper \%hash_net;
  my $time_before  = time();
  my @vmstat       = `$VMSTAT_CPU 2>>$error_log`;
  my $time_after   = time();
  my $skip_summary = 0;
  if ( $time_after - $time_before < $STEP / 2 ) {

    # workaround for old AIXes (5.2 etc) where is printed summary line at first an then actual data after specified time period
    # Linux use "vmstat 60 2" as default directly
    print "$VMSTAT_CPU 2>>$error_log\n";
    $VMSTAT_CPU =~ s/ 1$/ 2/;
    $skip_summary = 1;
    @vmstat       = `$VMSTAT_CPU 2>>$error_log`;
  }

  my $us_indx           = 0;
  my $sy_indx           = 0;
  my $wa_indx           = 0;
  my $pc_indx           = 0;
  my $st_indx           = 0;
  my $cpu_us            = -1;
  my $cpu_sy            = -1;
  my $cpu_wa            = -1;
  my $cpu_pc            = -1;
  my $cpu_st            = -1;
  my $linux_avoid_first = 0;

  if ( $os =~ m/SunOS/ ) {
    #
    #Solaris
    #
    foreach my $line (@vmstat) {
      chomp($line);
      my $indx = 0;
      #print "$line\n";
      my $i = 0;
      if ( $line =~ m/ us sy / ) {
        foreach my $item ( split( / +/, $line ) ) {
          $item =~ s/ //g;
          if ( $item eq '' ) {
            next;
          }
          #print "098 $items\n";
          #find index of user and sys time
          if ( $item =~ m/^us$/ ) {
            $us_indx = $indx;
          }
          if ( $item =~ m/^sy$/ ) {
            #if ( $i == 0 ) { $i++; next; }
            $sy_indx = $indx;
          }
          $indx++;
        }
        next;
      }

      #print "097 $line $us_indx - $sy_indx \n";
      if ( $us_indx > 0 && $sy_indx > 0 ) {
        if ( $os =~ m/Solaris/ && $linux_avoid_first == 0 ) {
          $linux_avoid_first = 1;
          next;
        }
        if ( $skip_summary == 1 ) {
          $skip_summary = 0;
          next;
        }
        $indx = 0;
        my $cpu_usr = "";
        my $cpu_sys = "";
        foreach my $item ( split( / +/, $line ) ) {
          $item =~ s/ //g;
          if ( $item eq '' ) {
            next;
          }
          if ( $indx == $us_indx ) {
            $cpu_usr = $item;
          }
          if ( $indx == $sy_indx ) {
            $cpu_sys = $item;
          }
          if ( $indx == $pc_indx ) {
            if ( !($wpar_id eq '') && isdigit($wpar_id) && $wpar_id > 0 ) {

              # co to je???
              $entitled = $cpu_pc;
            }
          }
          $indx++;
        }
        #####CPU load X
        $cpu_sy = $cpu_sys;
        $cpu_us = $cpu_usr;

        #print "001 $cpu_sy : $cpu_us : $us_indx : $sy_indx : $indx\n";
        # conversion to physical in rrdtool graphing when it appears necessary
        #$cpu_sy =  sprintf("%.2f",$entitled * $cpu_sy / 100);
        #$cpu_us =  sprintf("%.2f",$entitled * $cpu_us / 100);
        last;
      }
    }

    #$cpu_util = ":cpu:$entitled:$cpu_sy:$cpu_us:$cpu_wa";
    $message .= ":cpu:::$entitled:$cpu_sy:$cpu_us:0:$uptime_days:";
  }
  else {
    #
    #Linux and AIX
    #
    foreach my $line (@vmstat) {
      chomp($line);
      my $indx = 0;

      #print "099 $line\n";
      if ( $line =~ m/ us sy / ) {
        foreach my $item ( split( / +/, $line ) ) {
          $item =~ s/ //g;
          if ( $item eq '' ) {
            next;
          }

          #print "098 $item\n";
          # find index of user and sys time
          if ( $item =~ m/^us$/ ) {
            $us_indx = $indx;
          }
          if ( $item =~ m/^sy$/ ) {
            $sy_indx = $indx;
          }
          if ( $item =~ m/^wa$/ ) {
            $wa_indx = $indx;
          }
          if ( $item =~ m/^pc$/ ) {
            $pc_indx = $indx;
          }
          if ( $item =~ m/^st$/ ) {
            $st_indx = $indx;
          }
          $indx++;
        }
        next;
      }

      #print "099 $line $us_indx - $sy_indx \n";
      if ( $us_indx > 0 && $sy_indx > 0 ) {
        if ( $os =~ m/Linux/ && $linux_avoid_first == 0 ) {
          $linux_avoid_first = 1;
          next;
        }
        if ( $skip_summary == 1 ) {
          $skip_summary = 0;
          next;
        }
        $indx = 0;
        foreach my $item ( split( / +/, $line ) ) {
          $item =~ s/ //g;
          if ( $item eq '' ) {
            next;
          }
          if ( $indx == $us_indx ) {
            $cpu_us = $item;
          }
          if ( $indx == $sy_indx ) {
            $cpu_sy = $item;
          }
          if ( $indx == $st_indx ) {
            $cpu_st = $item;
          }
          if ( $indx == $wa_indx ) {
            $cpu_wa = $item if $item ne "-";
          }
          if ( $indx == $pc_indx ) {
            $cpu_pc = $item if $item ne "-";
            $cpu_pc = '' unless $os eq 'AIX';
            if ( !($wpar_id eq '') && isdigit($wpar_id) && $wpar_id > 0 ) {

              # WPARs: Use $entitled for passing "pc", vmstat does not show "id" & "wa"
              # use "pc" for something like standard CPU utilization obtained from the HMC for ordinary lpars
              # --> only for WPARs!!!
              $entitled = $cpu_pc;
            }
          }
          $indx++;
        }

        #print "001 $cpu_sy : $cpu_us : $us_indx : $sy_indx : $indx\n";
        # conversion to physical in rrdtool graphing when it appears necessary
        #$cpu_sy =  sprintf("%.2f",$entitled * $cpu_sy / 100);
        #$cpu_us =  sprintf("%.2f",$entitled * $cpu_us / 100);
        last;
      }
    }

    #$cpu_util = ":cpu:$entitled:$cpu_sy:$cpu_us:$cpu_wa";
    $message .= ":cpu::$cpu_pc:$entitled:$cpu_sy:$cpu_us:$cpu_wa:$uptime_days:";

    # CPU stolen is metric only for LINUX
    if ( $os =~ /Linux/ ) {
      if ( defined $cpu_st && $cpu_st > -1 ) {
        chomp $cpu_st;
        $cpu_st =~ s/\s+$//g;
        $message .= ":st-cpu:::$cpu_st:\::::";
      }
    }
  }

  #
  # SAN
  #

  # fcstat
  my @fcs_all;
  my $fcstat_al = 0;
  if ( defined $ENV{FCSTAT_AL} ) { $fcstat_al = $ENV{FCSTAT_AL}; }

  while ( my $line = shift @lsdev_c ) {
    chomp($line);
    if ( $line =~ m/^fcs/ ) {
      ( my $fcs, undef ) = split( / +/, $line );

      # check if the adapter is connected at first
      my $fcsi = $fcs;
      $fcsi =~ s/^fcs/fscsi/;
      if ( process_limits( "$FCSTAT_ALIVE $fcsi", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$FCSTAT_ALIVE $fcsi\", skipping it " . __FILE__ . ":" . __LINE__ );
        last;
      }
      print "$FCSTAT_ALIVE $fcsi 2>/dev/null \n";
      my $fcstat_alive = `$FCSTAT_ALIVE $fcsi 2>/dev/null`;    # bez error logging
      chomp($fcstat_alive);
      ( my $attach, my $attach_data ) = split( / +/, $fcstat_alive );
      if ( !defined($attach_data) || $attach_data eq '' || $attach_data =~ m/none/ || $attach_data =~ m/0x0/ ) {
        print "exluding $fcs, not connected : $fcstat_alive\n";
        next;
      }
      if ( $fcstat_al == 0 && isdigit($fcstat_al) && $attach_data =~ m/^al$/ ) {
        print "exluding $fcs, attached as arbitrary loop, use this to avoid it if you use FC arbitrary loop setup: \"FCSTAT_AL=1\" in OS agent env\n";
        next;
      }
      if ( process_limits( "$FCSTAT_ALIVE $fcs", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$FCSTAT_ALIVE $fcs\", skipping it " . __FILE__ . ":" . __LINE__ );
        last;
      }
      print "$FCSTAT -e $fcs 2>/dev/null\n";
      my @fcstat = `$FCSTAT -e $fcs 2>/dev/null`;
      my @fc_scsi_data;
      my @fc_nvme_data;
      my $found_scsi = 0;
      my $found_nvme = 0;
      foreach my $line (@fcstat) {

        if ($line =~ /FC SCSI Traffic Statistics/) {
          $found_scsi = 1;
          $found_nvme = 0;
          next;
        }

        if ($line =~ /FC NVME Traffic Statistics/) {
          $found_nvme = 1;
          $found_scsi = 0;
          next;
        }

        if ($line =~ /^[A-Z].+Statistics/ || $line =~ /^[A-Z].+Information/) {
          $found_scsi = 0;
          $found_nvme = 0;
        }

        push @fc_scsi_data, $line if $found_scsi;
        push @fc_nvme_data, $line if $found_nvme;
      }
      if (@fc_scsi_data){
        my $wwn = "";
        if ( ( grep( /World Wide Port Name:/, @fcstat ) ) ) {
          ($wwn) = grep /World Wide Port Name:/, @fcstat;
          $wwn =~ s/World Wide Port Name:\s*//;
          $wwn =~ s/\s+$//g;
          $wwn =~ s/0x//g;
        }
        my ($input_bytes,$output_bytes,$in_req,$out_req) = "";
        my ($nvme_input_bytes,$nvme_output_bytes,$nvme_in_req,$nvme_out_req) = "";
        if ( ( grep( /Input Bytes:/, @fc_scsi_data ) ) ) {
          ($input_bytes)       = grep /Input Bytes:/, @fc_scsi_data;
          if ($input_bytes){
            $input_bytes    =~ s/Input Bytes://g;
            $input_bytes    =~ s/^\s+//g;
            $input_bytes    =~ s/\s+$//g;
            if ( ! isdigit($input_bytes) ) {
              $input_bytes = 0;
            }
            else{
              if ($input_bytes <= 0){
                $input_bytes = 0;
              }
            }
          }
        }
        if ( ( grep( /Output Bytes:/, @fc_scsi_data ) ) ) {
          ($output_bytes)       = grep /Output Bytes:/, @fc_scsi_data;
          if ($output_bytes){
            $output_bytes    =~ s/Output Bytes://g;
            $output_bytes    =~ s/^\s+//g;
            $output_bytes    =~ s/\s+$//g;
            if ( ! isdigit($output_bytes) ) {
              $output_bytes = 0;
            }
            else{
              if ($output_bytes <= 0){
                $output_bytes = 0;
              }
            }
          }
        }
        if ( ( grep( /Input Requests:/, @fc_scsi_data ) ) ) {
          ($in_req)       = grep /Input Requests:/, @fc_scsi_data;
          if ($in_req){
            $in_req    =~ s/Input Requests://g;
            $in_req    =~ s/^\s+//g;
            $in_req    =~ s/\s+$//g;
            if ( ! isdigit($in_req) ) {
              $in_req = 0;
            }
            else{
              if ($in_req <= 0){
                $in_req = 0;
              }
            }
          }
        }
        if ( ( grep( /Output Requests:/, @fc_scsi_data ) ) ) {
          ($out_req)       = grep /Output Requests:/, @fc_scsi_data;
          if ($out_req){
            $out_req    =~ s/Output Requests://g;
            $out_req    =~ s/^\s+//g;
            $out_req    =~ s/\s+$//g;
            if ( ! isdigit($out_req) ) {
              $out_req = 0;
            }
            else{
              if ($out_req <= 0){
                $out_req = 0;
              }
            }
          }
        }
        # NVME SECTION
        if ( grep(/Input Bytes:/, @fc_nvme_data) ) {
          ($nvme_input_bytes) = grep /Input Bytes:/, @fc_nvme_data;
          $nvme_input_bytes =~ s/Input Bytes://;
          $nvme_input_bytes =~ s/^\s+|\s+$//g;
          $nvme_input_bytes = 0 if !isdigit($nvme_input_bytes) || $nvme_input_bytes <= 0;
        }

        if ( grep(/Output Bytes:/, @fc_nvme_data) ) {
          ($nvme_output_bytes) = grep /Output Bytes:/, @fc_nvme_data;
          $nvme_output_bytes =~ s/Output Bytes://;
          $nvme_output_bytes =~ s/^\s+|\s+$//g;
          $nvme_output_bytes = 0 if !isdigit($nvme_output_bytes) || $nvme_output_bytes <= 0;
        }

        if ( grep(/Input Requests:/, @fc_nvme_data) ) {
          ($nvme_in_req) = grep /Input Requests:/, @fc_nvme_data;
          $nvme_in_req =~ s/Input Requests://;
          $nvme_in_req =~ s/^\s+|\s+$//g;
          $nvme_in_req = 0 if !isdigit($nvme_in_req) || $nvme_in_req <= 0;
        }

        if ( grep(/Output Requests:/, @fc_nvme_data) ) {
          ($nvme_out_req) = grep /Output Requests:/, @fc_nvme_data;
          $nvme_out_req =~ s/Output Requests://;
          $nvme_out_req =~ s/^\s+|\s+$//g;
          $nvme_out_req = 0 if !isdigit($nvme_out_req) || $nvme_out_req <= 0;
        }
        $message  .= ":san:$fcs:$wwn:$input_bytes:$output_bytes:$in_req:$out_req\::";
        # FCS nvme new section
        if (@fc_nvme_data) {
          if ( $nvme_input_bytes > 0 || $nvme_output_bytes > 0 || $nvme_in_req > 0 || $nvme_out_req > 0 ) {
            $message  .= ":san:$fcs-nvme:$wwn:$nvme_input_bytes:$nvme_output_bytes:$nvme_in_req:$nvme_out_req\::";
          }
        }
      }
      my ($lip_count,$nos_count,$error_frames,$dumped_frames,$fail_count,$loss_sync_count,$loss_signal,$error_count,$txt_word,$fcs_error) = "";
      if ( ( grep( /LIP Count:/, @fcstat ) ) ) {
        ($lip_count)       = grep /LIP Count:/, @fcstat;
        if ($lip_count){
          $lip_count    =~ s/LIP Count://g;
          $lip_count    =~ s/^\s+//g;
          $lip_count    =~ s/\s+$//g;
          if ( ! isdigit($lip_count) ) {
            $lip_count = 0;
          }
          else{
            if ($lip_count <= 0){
              $lip_count = 0;
            }
          }
        }
      }
      else { $lip_count = 0 }
      if ( ( grep( /NOS Count:/, @fcstat ) ) ) {
        ($nos_count)       = grep /NOS Count:/, @fcstat;
        if ($nos_count){
          $nos_count    =~ s/NOS Count://g;
          $nos_count    =~ s/^\s+//g;
          $nos_count    =~ s/\s+$//g;
          if ( ! isdigit($nos_count) ) {
            $nos_count = 0;
          }
          else{
            if ($nos_count <= 0){
              $nos_count = 0;
            }
          }
        }
      }
      else { $nos_count = 0 }
      if ( ( grep( /Error Frames:/, @fcstat ) ) ) {
        ($error_frames)    = grep /Error Frames:/, @fcstat;
        if ($error_frames){
          $error_frames    =~ s/Error Frames://g;
          $error_frames    =~ s/^\s+//g;
          $error_frames    =~ s/\s+$//g;
          if ( ! isdigit($error_frames) ) {
            $error_frames = 0;
          }
          else{
            if ($error_frames <= 0){
              $error_frames = 0;
            }
          }
        }
      }
      else { $error_frames = 0 }
      if ( ( grep( /Dumped Frames:/, @fcstat ) ) ) {
        ($dumped_frames)   = grep /Dumped Frames:/, @fcstat;
        if ($dumped_frames){
          $dumped_frames    =~ s/Dumped Frames://g;
          $dumped_frames    =~ s/^\s+//g;
          $dumped_frames    =~ s/\s+$//g;
          if ( ! isdigit($dumped_frames) ) {
            $dumped_frames = 0;
          }
          else{
            if ($dumped_frames <= 0){
              $dumped_frames = 0;
            }
          }
        }
      }
      else { $dumped_frames = 0 }
      if ( ( grep( /Link Failure Count:/, @fcstat ) ) ) {
        ($fail_count)      = grep /Link Failure Count:/, @fcstat;
        if ($fail_count){
          $fail_count    =~ s/Link Failure Count://g;
          $fail_count    =~ s/^\s+//g;
          $fail_count    =~ s/\s+$//g;
          if ( ! isdigit($fail_count) ) {
            $fail_count = 0;
          }
          else{
            if ($fail_count <= 0){
              $fail_count = 0;
            }
          }
        }
      }
      else { $fail_count = 0 }
      if ( ( grep( /Loss of Sync Count:/, @fcstat ) ) ) {
        ($loss_sync_count) = grep /Loss of Sync Count:/, @fcstat;
        if ($loss_sync_count){
          $loss_sync_count    =~ s/Loss of Sync Count://g;
          $loss_sync_count    =~ s/^\s+//g;
          $loss_sync_count    =~ s/\s+$//g;
          if ( ! isdigit($loss_sync_count) ) {
            $loss_sync_count = 0;
          }
          else{
            if ($loss_sync_count <= 0){
              $loss_sync_count = 0;
            }
          }
        }
      }
      else { $loss_sync_count = 0 }
      if ( ( grep( /Loss of Signal:/, @fcstat ) ) ) {
        ($loss_signal)     = grep /Loss of Signal:/, @fcstat;
        if ($loss_signal){
          $loss_signal    =~ s/Loss of Signal://g;
          $loss_signal    =~ s/^\s+//g;
          $loss_signal    =~ s/\s+$//g;
          $loss_signal    = -1;
          if ( ! isdigit($loss_signal) ) {
            $loss_signal = 0;
          }
          else{
            if ($loss_signal <= 0){
              $loss_signal = 0;
            }
          }
        }
      }
      else { $loss_signal = 0 }
      if ( ( grep( /Primitive Seq Protocol Error Count:/, @fcstat ) ) ) {
        ($error_count)     = grep /Primitive Seq Protocol Error Count:/, @fcstat;
        if ($error_count){
          $error_count    =~ s/Primitive Seq Protocol Error Count://g;
          $error_count    =~ s/^\s+//g;
          $error_count    =~ s/\s+$//g;
          if ( ! isdigit($error_count) ) {
            $error_count = 0;
          }
          else{
            if ($error_count <= 0){
              $error_count = 0;
            }
          }
        }
      }
      else { $error_count = 0 }
      if ( ( grep( /Invalid Tx Word Count:/, @fcstat ) ) ) {
        ($txt_word)        = grep /Invalid Tx Word Count:/, @fcstat;
        if ($txt_word){
          $txt_word    =~ s/Invalid Tx Word Count://g;
          $txt_word    =~ s/^\s+//g;
          $txt_word    =~ s/\s+$//g;
          if ( ! isdigit($txt_word) ) {
            $txt_word = 0;
          }
          else{
            if ($txt_word <= 0){
              $txt_word = 0;
            }
          }
        }
      }
      else { $txt_word = 0 }
      if ( ( grep( /Invalid CRC Count:/, @fcstat ) ) ) {
        ($fcs_error)       = grep /Invalid CRC Count:/, @fcstat;
        if ($fcs_error){
          $fcs_error    =~ s/Invalid CRC Count://g;
          $fcs_error    =~ s/^\s+//g;
          $fcs_error    =~ s/\s+$//g;
          if ( ! isdigit($fcs_error) ) {
            $fcs_error = 0;
          }
          else{
            if ($fcs_error <= 0){
              $fcs_error = 0;
            }
          }
        }
      }
      else { $fcs_error = 0 }
      my $fcs_err_total = $lip_count + $nos_count + $error_frames + $dumped_frames + $fail_count + $loss_sync_count + $loss_signal + $error_count + $txt_word + $fcs_error;
      if ($fcs_err_total > 0){
        $message .= ":san_error:$fcs:$fcs_err_total:\:::::";
      }
      my $tx_power = "";
      my $rx_power = "";
      if ( ( grep( /TX Power:/, @fcstat ) ) ) {
        ($tx_power)       = grep /TX Power:/, @fcstat;
        if ($tx_power){
          $tx_power    =~ s/TX Power://g;
          $tx_power    =~ s/\[.*?\]//g;
          $tx_power    =~ s/dBm//;
          $tx_power    =~ s/^\s+//g;
          $tx_power    =~ s/\s+$//g;
        }
      }
      else { $tx_power = 0 }
      if ( ( grep( /RX Power:/, @fcstat ) ) ) {
        ($rx_power)       = grep /RX Power:/, @fcstat;
        if ($rx_power){
          $rx_power    =~ s/RX Power://g;
          $rx_power    =~ s/\[.*?\]//g;
          $rx_power    =~ s/dBm//;
          $rx_power    =~ s/^\s+//g;
          $rx_power    =~ s/\s+$//g;
        }
      }
      else { $rx_power = 0 }
      if (defined $tx_power && $tx_power ne "" && $tx_power ne "UNKNOWN" && defined $rx_power && $rx_power ne "" && $rx_power ne "UNKNOWN") {
        #print ":san_power:$fcs:$tx_power:$rx_power:\::::";
        $message .= ":san_power:$fcs:$tx_power:$rx_power:\::::";
      }
    }
  }

  # Linux
  my $fcs_path = "/sys/class/fc_host";
  if ( -d $fcs_path ) {

    # FC adapters exist
    opendir( DIR, $fcs_path ) || error( "Directory cannot be open : $fcs_path : $! " . __FILE__ . ":" . __LINE__ );
    my @fcadapters = readdir(DIR);
    closedir(DIR);

    foreach my $fcs (@fcadapters) {
      chomp($fcs);
      my $tx_frames = "";
      my $rx_frames = "";
      my $tx_words  = "";
      my $rx_words  = "";
      my $wwn       = "";

      if ( -f "$fcs_path/$fcs/statistics/tx_frames" ) {
        open( FH, "< $fcs_path/$fcs/statistics/tx_frames" ) || error( " Can't open $fcs_path/$fcs/statistics/tx_frames: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my $data = <FH>;
        chomp($data);
        close(FH);
        if ( ishexa($data) == 1 ) {
          $tx_frames = sprintf( "%d", hex($data) );
        }
      }
      if ( -f "$fcs_path/$fcs/statistics/rx_frames" ) {
        open( FH, "< $fcs_path/$fcs/statistics/rx_frames" ) || error( " Can't open $fcs_path/$fcs/statistics/rx_frames: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my $data = <FH>;
        chomp($data);
        close(FH);
        if ( ishexa($data) == 1 ) {
          $rx_frames = sprintf( "%d", hex($data) );
        }
      }
      if ( -f "$fcs_path/$fcs/statistics/fcp_input_megabytes" ) {
        open( FH, "< $fcs_path/$fcs/statistics/fcp_input_megabytes" ) || error( " Can't open $fcs_path/$fcs/statistics/fcp_input_megabytes: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my $data = <FH>;
        chomp($data);
        close(FH);
        if ($data eq "0x0"){
          if ( -f "$fcs_path/$fcs/statistics/rx_words" ) {
            open( FH, "< $fcs_path/$fcs/statistics/rx_words" ) || error( " Can't open $fcs_path/$fcs/statistics/rx_words: $! " . __FILE__ . ":" . __LINE__ ) && next;
            my $data_rx = <FH>;
            chomp($data_rx);
            close(FH);
            $data_rx = 1 if ( $data_rx eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
            if ( ishexa($data_rx) == 1 ) {
              $rx_words = sprintf( "%d", hex($data_rx) ) * 4;     # convert to Bytes
            }
          }
        }
        else{
          $data = 1 if ( $data eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
          if ( ishexa($data) == 1 ) {
            $rx_words = sprintf( "%d", hex($data) ) * 1048576; # # convert MB to Bytes
          }
        }
      }
      else{
        if ( -f "$fcs_path/$fcs/statistics/rx_words" ) {
          open( FH, "< $fcs_path/$fcs/statistics/rx_words" ) || error( " Can't open $fcs_path/$fcs/statistics/rx_words: $! " . __FILE__ . ":" . __LINE__ ) && next;
          my $data_rx = <FH>;
          chomp($data_rx);
          close(FH);
          $data_rx = 1 if ( $data_rx eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
          if ( ishexa($data_rx) == 1 ) {
            $rx_words = sprintf( "%d", hex($data_rx) ) * 4;     # convert to Bytes
          }
        }
      }
      if ( -f "$fcs_path/$fcs/statistics/fcp_output_megabytes" ) {
        open( FH, "< $fcs_path/$fcs/statistics/fcp_output_megabytes" ) || error( " Can't open $fcs_path/$fcs/statistics/fcp_output_megabytes: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my $data = <FH>;
        chomp($data);
        close(FH);
        if ($data eq "0x0"){
          if ( -f "$fcs_path/$fcs/statistics/tx_words" ) {
            open( FH, "< $fcs_path/$fcs/statistics/tx_words" ) || error( " Can't open $fcs_path/$fcs/statistics/tx_words: $! " . __FILE__ . ":" . __LINE__ ) && next;
            my $data_tx = <FH>;
            chomp($data_tx);
            close(FH);
            $data_tx = 1 if ( $data_tx eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
            if ( ishexa($data_tx) == 1 ) {
              $tx_words = sprintf( "%d", hex($data_tx) ) * 4;     # convert to Bytes
            }
          }
        }
        else{
          $data = 1 if ( $data eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
          if ( ishexa($data) == 1 ) {
            $tx_words = sprintf( "%d", hex($data) ) * 1048576;   # convert MB to Bytes
          }
        }
      }
      else{
        if ( -f "$fcs_path/$fcs/statistics/tx_words" ) {
          open( FH, "< $fcs_path/$fcs/statistics/tx_words" ) || error( " Can't open $fcs_path/$fcs/statistics/tx_words: $! " . __FILE__ . ":" . __LINE__ ) && next;
          my $data_tx = <FH>;
          chomp($data_tx);
          close(FH);
          $data_tx = 1 if ( $data_tx eq '0xffffffffffffffff' );    # RHEL 5 does not provide tx_words due to a bug, just workaround to exclude it
          if ( ishexa($data_tx) == 1 ) {
            $tx_words = sprintf( "%d", hex($data_tx) ) * 4;     # convert to Bytes
          }
        }
      }
      # WWN fc port
      if ( -f "$fcs_path/$fcs/port_name" ) {
        open( FH, "< $fcs_path/$fcs/port_name" ) || error( " Can't open $fcs_path/$fcs/port_name: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my $data = <FH>;
        chomp($data);
        close(FH);
        $wwn = $data;
        $wwn =~ s/\s+$//g;
        $wwn =~ s/0x//g;
      }
      if ( !($tx_frames eq '') && !($rx_frames eq '') && !($tx_words eq '') && !($rx_words eq '') ) {
        if ( isdigit($tx_frames) && isdigit($rx_frames) && isdigit($tx_words) && isdigit($rx_words) && $tx_frames > 0 && $rx_frames > 0 && $tx_words > 0 && $rx_words > 0 ) {
          $message .= ":san:$fcs:$wwn:$rx_words:$tx_words:$tx_frames:$rx_frames\::";

          # $tx_frames:$rx_frames are swaped here, this is corected in detail-graph.pl since 5.07-04
        }
      }
    }
  }

  # wait fork iostat
  if ( $os =~ /AIX|SunOS/ ) {
    waitpid( $pid_iostat, 0 );
    if ( -f $iostat_data_file ) {
      open( IO, "< $iostat_data_file" ) || error( "Cannot open $iostat_data_file: $!" . __FILE__ . ":" . __LINE__ );
      my @adapters_data = <IO>;
      close(IO);
      unlink($iostat_data_file);
      foreach my $line (@adapters_data) {
        chomp $line;
        my ( $ada_name, $bytes_sec_r, $bytes_sec_w, $io_read, $io_write, $resp_time_read, $resp_time_write ) = split( ",", $line );
        if ($resp_time_read){
          chomp($resp_time_read);
        }
        if ($resp_time_write){
          chomp($resp_time_write);    # just to be sure
        }
        if ( $os eq "SunOS" && $lpar_id =~ /cdom|ldom/ ) {
          my @ldom_info1 = "";
          if ( $type_of_solaris =~ /sparc/ ) {
            if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
              error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
            }
            else {
              @ldom_info1 = `$VIRTINFO 2>>$error_log`;
            }
          }
          if ( grep /Domain UUID:/, @ldom_info1 ) {
            my ($ldom_uuid1) = grep /Domain UUID:/, @ldom_info1;
            $ldom_uuid1 =~ s/Domain UUID://g;
            $ldom_uuid1 =~ s/\s+//g;
            chomp $ldom_uuid1;
            $message .= ":san_l:$ldom_uuid1:$ada_name:$io_read:$io_write:$bytes_sec_r:$bytes_sec_w\:::san_tresp:$ldom_uuid1:$ada_name:$resp_time_read:\::::";
          }
        }
        if ( $os eq "SunOS" && $lpar_id =~ /no_ldom|zone/ ) {
          my $hostname = `hostname`;
          chomp $hostname;
          $message .= ":san_l:$hostname:$ada_name:$io_read:$io_write:$bytes_sec_r:$bytes_sec_w\:::san_tresp:$hostname:$ada_name:$resp_time_read:\::::";
        }
        if ( grep( /^$ada_name,/, @fcs_all ) ) {

          # send only response times here
          $message .= ":san_resp:$ada_name\::$resp_time_read:$resp_time_write\::::";
        }
        else {
          if ( $ada_name =~ m/^fcs/ ) {

            # never send data for FCS adapters from iostat as fcstat use COUNTER and iostat GAUGE!
            $message .= ":san_resp:$ada_name\::$resp_time_read:$resp_time_write\::::";
          }
          else {
            if ( $os ne "SunOS" ) {
              $message .= ":san:$ada_name\::$bytes_sec_r:$bytes_sec_w:$io_read:$io_write\:::san_resp:$ada_name\::$resp_time_read:$resp_time_write\::::";
            }
          }
        }
      }
    }
  }

  #
  #
  # VMSTAT IOSTAT/DATA
  #
  # difference counting

  if ( $os =~ /Linux/ ) {
    my $time_after_iostat;
    my @vmstat_io;
    my @all_disk;
    if ( process_limits( "$VMSTAT_IOSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$VMSTAT_IOSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @vmstat_io         = `$VMSTAT_IOSTAT 2>>$error_log`;
      $time_after_iostat = time();
    }
    if ( process_limits( "$LIST_DISK", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$VMSTAT_IOSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @all_disk = `$LIST_DISK 2>>$error_log`;
    }
    my @list_all_disk;
    foreach my $disk (@all_disk) {
      if ( $disk =~ /^NAME/ ) {next}
      #my ($disk_name) = grep /^[hsv]d[a-z] |^nvme\dn\d |^xvd[a-z] /, $disk;    # the better way getting list of disk, because of older Linux systems
      my ($disk_name) = grep /^[hsv]d[a-z] |^[hsv]d[a-z][a-z] |^nvme\dn\d |^nvme\dn\d\d |^xvd[a-z] |^xvd[a-z][a-z] /, $disk;    # the better way getting list of disk, because of older Linux systems
       # old ATA (hda, hdb, etc.) and NVME SSD (nvme0n1, etc.) disks added [JD]
       # added vd* xvd* -PH : Xen paravirt drivers use /dev/xvda (first detected disk) and the standard Linux virtio drivers use /dev/vda (first detected disk) for example.

      #my ($disk_name,undef,undef,undef,undef,$type) = split( " ", $disk );
      #if ($type ne "disk"){next}
      if ( defined $disk_name ) {
      push @list_all_disk, $disk_name;
      }

      #print "$disk_name,$type\n";
    }

    # wait fork vmstat_iostat
    my ( $iops_read_now, $data_read_now, $latency_read_now, $iops_write_now, $data_write_now, $latency_write_now );
    my ( $iops_read_bef, $data_read_bef, $latency_read_bef, $iops_write_bef, $data_write_bef, $latency_write_bef );    # data from first run
    my ( $total_read, $total_iops_read, $total_write, $total_iops_write );
    my @iostat_data = @iostat_data_60sec_before;

    my ( $iops_read_res, $data_read_res, $latency_read_res, $iops_write_res, $data_write_res, $latency_write_res, $data_total_read, $data_total_write );

    # data now minus data from first run
    foreach my $line (@vmstat_io) {
      chomp $line;
      my ( $disk_name, $iops_read, undef, $data_read, $ms_read, $iops_write, undef, $data_write, $ms_write ) = split( " ", $line );
      my $if_disk = grep {/$disk_name/} @list_all_disk;
      if ( $if_disk != 1 ) {next}
      my ($grep_line) = grep {/$disk_name/} @iostat_data;
      my ( undef, $item, $disk_name1, $iops_read1, $data_read1, $ms_read1, $iops_write1, $data_write1, $ms_write1 ) = split( /:/, $grep_line );    # data from first run
      my $HW_SECTOR_SIZE = "cat /sys/block/$disk_name1/queue/hw_sector_size";
      my $sector_size;

      if ( !-f $HW_SECTOR_SIZE ) {
        if ( process_limits( "$HW_SECTOR_SIZE", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$HW_SECTOR_SIZE\", skipping it " . __FILE__ . ":" . __LINE__ );
        }
        else {
          $sector_size = `$HW_SECTOR_SIZE 2>/dev/null`;
          if ($sector_size) {
            chomp $sector_size;
          }
        }
      }
      else {
        $sector_size = 512;    # default in most of cases
      }
      chomp( $iops_read1, $iops_write1 );
      my $iops_read_diff  = $iops_read - $iops_read1;
      my $iops_write_diff = $iops_write - $iops_write1;
      my $ms_read_diff    = $ms_read - $ms_read1;
      my $ms_write_diff   = $ms_write - $ms_write1;
      my $time_io         = $time_after_iostat - $time_before_iostat;    # time how long iostat running
      $data_read_res  = ( $data_read - $data_read1 ) * $sector_size;
      $data_write_res = ( $data_write - $data_write1 ) * $sector_size;

      my $iops_readxms_read   = $iops_read_diff * $ms_read_diff;         # IOPS x ms latency
      my $iops_writexms_write = $iops_write_diff * $ms_write_diff;       # IOPS x ms latency

      $iops_read_now     += $iops_read;
      $iops_write_now    += $iops_write;
      $latency_read_now  += $ms_read;
      $latency_write_now += $ms_write;

      # latency counting
      $total_read       += $iops_readxms_read;
      $total_iops_read  += $iops_read_diff;
      $total_write      += $iops_writexms_write;
      $total_iops_write += $iops_write_diff;
      $data_total_read  += $data_read_res;
      $data_total_write += $data_write_res;

      # data before 60 sec
      $iops_read_bef     += $iops_read1;
      $iops_write_bef    += $iops_write1;
      $latency_read_bef  += $ms_read1;
      $latency_write_bef += $ms_write1;

    }
    $iops_read_res     = $iops_read_now - $iops_read_bef;
    $iops_write_res    = $iops_write_now - $iops_write_bef;
    $latency_read_res  = $latency_read_now - $latency_read_bef;
    $latency_write_res = $latency_write_now - $latency_write_bef;
    if ( $iops_read_res != 0 ) {
      $latency_read_res = $latency_read_res / $iops_read_res;
    }
    if ( $iops_write_res != 0 ) {
      $latency_write_res = $latency_write_res / $iops_write_res;
    }
    chomp( $iops_read_res, $data_read_res, $latency_read_res, $iops_write_res, $data_write_res, $latency_write_res );
    my $time_io = $time_after_iostat - $time_before_iostat;    # time how long iostat running
                                                               #print "BEFORE / ($time_io): $iops_read_res,$data_read_res,$latency_read_res,$iops_write_res,$data_write_res,$latency_write_res\n";
    if ( isdigit($time_io) && $time_io != 0 ) {
        $iops_read_res    = $iops_read_res / $time_io;
        $iops_write_res   = $iops_write_res / $time_io;
        $data_total_read  = $data_total_read / $time_io;
        $data_total_write = $data_total_write / $time_io;

        #print "IOPS:$iops_read_res,$iops_write_res\n";
        #print "LATENCY:$latency_read_res,$latency_write_res\n";
        $iops_read_res     = sprintf( "%.2f", $iops_read_res );
        $data_total_read   = sprintf( "%.2f", $data_total_read );
        $iops_write_res    = sprintf( "%.2f", $iops_write_res );
        $data_total_write  = sprintf( "%.2f", $data_total_write );
        $latency_read_res  = sprintf( "%.2f", $latency_read_res );
        $latency_write_res = sprintf( "%.2f", $latency_write_res );
        $message .= ":disk-total:$iops_read_res:$data_total_read:$latency_read_res:$iops_write_res:$data_total_write:$latency_write_res:\:";
    }
  }

  # wait fork wlmstat
  if ( $os =~ /AIX/ && ( $wpar_id eq '' || !($wpar_id > 0) || !isdigit($wpar_id) ) ) {

    # Wait for wlmstat fork
    waitpid( $pid_wlmstat, 0 );
    if ( -f $wlmstat_data_file ) {
      open( WLM, "< $wlmstat_data_file" ) || error( "Cannot open $wlmstat_data_file\: $!" . __FILE__ . ":" . __LINE__ );
      my @class_data = <WLM>;
      close(WLM);
      unlink($wlmstat_data_file);
      foreach my $line (@class_data) {
        chomp $line;
        if ( $line eq '' ) {
          next;
        }
        my ( $class, $cpu, $mem, $dkio, $class_type ) = split( "-", $line );
        $message .= ":wlm:$class\:$cpu\:$mem\:$dkio\:$class_type\:::";
      }
      print "Wlmstat fork ended successfully \n" if $DEBUG == 3;
    }
  }

  # wait fork prstat
  if ( $os =~ /SunOS/ && $grep_s eq "5.10" ) {
    waitpid( $pid_prstat, 0 );
    if ( -f $prstat_data_file ) {
      open( PRS, "< $prstat_data_file" ) || error( "Cannot open $prstat_data_file\: $!" . __FILE__ . ":" . __LINE__ );
      my @prstat_data = <PRS>;
      close(PRS);
      unlink($prstat_data_file);
      foreach my $line (@prstat_data) {
        chomp $line;
        $message .= "$line";
      }
    }
  }

  #wait fork zonestat
  if ( $os =~ /SunOS/ && $grep_s eq "5.11" ) {
    my %zone_uuid;
    my @list_of_zone_uuid = `$ZONEADM_P`;
    foreach my $l (@list_of_zone_uuid) {
      chomp $l;
      my @f = split /:/, $l;
      $zone_uuid{$f[1]} = $f[4] if defined $f[4];
    }
    my $zone_name1 = `zonename`;
    chomp $zone_name1;
    waitpid( $pid_zonestat, 0 );
    if ( -f $zonestat_data_file ) {
      open my $ZON, "<", $zonestat_data_file or error("Cannot open $zonestat_data_file: $!");
      my $memory_phys = "";
      # Resource memory
      while ( my $line = <$ZON> ) {
        if ( $line =~ /physical-memory:mem_default:\[resource\]/ ) {
          my @f = split /:/, $line;
          $memory_phys = $f[5];
          $memory_phys =~ s/K//;
          chomp $memory_phys;
          last;
        }
      }
      seek($ZON, 0, 0);
      while ( my $line = <$ZON> ) {
        chomp $line;
        next unless $line =~ /interval:summary:/;
        my @f = split /:/, $line;
        my $zone_name = $f[3];
        $zone_name =~ s/[\[\]]//g;
        next unless $zone_name1 eq "global";

        my $uuid_zone;

        # total + system -> LDOM UUID (total and system are not in the zoneadm list)
        my $ldom_uuid_zone = "";
        my $char_l         = "";
        if ( $zone_name eq 'total' || $zone_name eq 'system' ) {
          $uuid_zone = $ldom_uuid_zone;
        }
        else {
          $uuid_zone = $zone_uuid{$zone_name};
        }

        next unless defined $uuid_zone;

        my ($uuid_zone_grep)      = grep ( /$zone_name/, @list_of_zone_uuid );
        my $cpu_used              = $f[4];
        my $cpu_used_in_perc      = $f[5];  $cpu_used_in_perc =~ s/%//;
        my $phys_mem_used         = $f[8];  $phys_mem_used =~ s/K//;
        my $phys_mem_used_in_perc = $f[9];  $phys_mem_used_in_perc =~ s/%//;
        my $cap_mem_used_in_perc  = $f[10]; $cap_mem_used_in_perc =~ s/%//; $cap_mem_used_in_perc  =~ s/-/U/;
        my $virt_mem_used_in_perc = $f[12]; $virt_mem_used_in_perc =~ s/%//;
        my $physnet_used_in_perc  = $f[15]; $physnet_used_in_perc =~ s/%//; $physnet_used_in_perc  =~ s/-/U/; $physnet_used_in_perc  =~ s/K//;
        my @ldom_info3 = "";

        if ( $type_of_solaris =~ /sparc/ ) {
          if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            @ldom_info3 = `$VIRTINFO 2>>$error_log`;
          }
        }
        if ( grep /Domain UUID:/, @ldom_info3 ) {
          ($ldom_uuid_zone) = grep /Domain UUID:/, @ldom_info3;
          $ldom_uuid_zone =~ s/Domain UUID://g;
          $ldom_uuid_zone =~ s/\s+//g;
          $char_l = "/";
          chomp $ldom_uuid_zone;
        }
        $message .= ":$zone_name$char_l$ldom_uuid_zone:$cap_mem_used_in_perc:$memory_phys:$cpu_used:$cpu_used_in_perc:$phys_mem_used:$phys_mem_used_in_perc:$virt_mem_used_in_perc:$physnet_used_in_perc";
      }
      close $ZON;
      unlink $zonestat_data_file;
    }
  }


  #wait fork poolstat
  if ( $os =~ /SunOS/ && $grep_s =~ /5\.10|5\.11/ ) {
    waitpid( $pid_poolstat, 0 );
    my $zone_name1 = `zonename 2>/dev/null`;
    chomp $zone_name1;
    if ( $zone_name1 eq "global" ) {
      if ( -f $poolstat_data_file ) {
        open( POOL, "< $poolstat_data_file" ) || error( "Cannot open $poolstat_data_file\: $!" . __FILE__ . ":" . __LINE__ );
        my @poolstat_data = <POOL>;
        close(POOL);
        unlink($poolstat_data_file);
        my %last_sample;
        foreach my $line (@poolstat_data) {
          chomp $line;

          next if $line =~ /^\s*$/;
          next if $line =~ /^\s*pset/;
          next if $line =~ /^\s*id\s+pool/;

          $line =~ s/^\s+//;
          $line =~ s/\s+/:/g;

          my ( $id, $pool_name, $size, $used, $load ) = split /:/, $line;

          $last_sample{$pool_name} = [$id, $size, $used, $load];
        }
        foreach my $pool (keys %last_sample) {
          my ($id, $size, $used, $load) = @{ $last_sample{$pool} };
          $message .= ":pool-sol:$pool:$id:$size:$used:$load:\::";
        }
      }
    }
  }
  #
  # LDOM PART - Solaris 10 and 11 - control and normal ldoms etc.
  #

  if ( $os =~ m/SunOS/ ) {
    my @smbios  = "";
    my $vm_name = `uname -n 2>>/dev/null`;

    #my @list_of_zone_uuid = "";
    if ( process_limits( "$ZONEADM_P", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$ZONEADM_P\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @list_of_zone_uuid = `$ZONEADM_P 2>>$error_log`;
    }
    chomp $vm_name;
    my @ldom_info1 = "";
    if ( $type_of_solaris =~ /sparc/ ) {
      if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        @ldom_info1 = `$VIRTINFO 2>>$error_log`;
      }
    }
    my $ldom_stat = "";
    if ( $role_of_ldom =~ /LDoms control/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        $ldom_stat = `$LDMLS 2>>$error_log`;
      }
    }
    my ( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util, $zone_name, $zone_grep ) = "";
    foreach my $domain ( split( /\n/, $ldom_stat ) ) {
      if ( $role_of_ldom =~ /LDoms control/ ) {
        if ( $domain =~ /DOMAIN|name=/ ) {
          ( undef, $ldom_name, $status, undef, undef, $vcpu, $all_memory, $util, $uptime, $norm_util ) = split( /\|/, $domain );
          $ldom_name  =~ s/name=//g;
          $status     =~ s/state=//g;
          $vcpu       =~ s/ncpu=//g;
          $all_memory =~ s/mem=//g;
          $util       =~ s/util=//g;
          $uptime     =~ s/uptime=//g;
          $norm_util  =~ s/norm_util=//g;
          chomp( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util );
          my @ldom_info2 = `$LDMINFO $ldom_name 2>>$error_log`;
          my ($uuid_ldom) = grep /uuid=/, @ldom_info2;
          $uuid_ldom =~ s/UUID\|uuid=//g;
          chomp $uuid_ldom;
          my @ldomlist = `$LDMLIST $ldom_name 2>>$error_log`;
          my ($ldom_hostname_id) = grep /HOSTID/, @ldomlist;
          $ldom_hostname_id =~ s/HOSTID\|hostid=0x//g;
          chomp $ldom_hostname_id;

          my $old_name = "$ldom_name";    #### primary and secondary ldom_name changed name
          if ( $status ne "active" ) {next}
          my $ldom_string = "_ldom";
          if ( $ldom_name =~ /primary/ ) {
            my $host_name = `hostname`;
            chomp $host_name;
            $ldom_name = "$host_name";
          }
          if ( $ldom_name =~ /secondary/ ) {
            my $host_name = `hostname`;
            chomp $host_name;
            $host_name =~ s/-primary//g;
            $host_name =~ s/-control//g;
            $ldom_name = "$host_name-secondary";
          }

          #$message .= ":$ldom_name$ldom_string:$uuid_ldom$char_l$ldom_hostname_id:$status:$vcpu:$all_memory:$util:$uptime:$norm_util\:";
          my $timestamp_after1 = time();
          my $net_stat         = "";
          if ( process_limits( "$LDMLSNET", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLSNET\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            $net_stat = `$LDMLSNET $old_name`;
          }
          foreach my $net_ldom ( split /\n/, $net_stat ) {
            if ( $net_ldom =~ /link/ ) {
              my ( $link_name, $ipackets, $rbytes, $opackets, $obytes ) = split( /\|/, $net_ldom );
              $link_name =~ s/link=//g;
              $ipackets  =~ s/ipackets=//g;
              $rbytes    =~ s/rbytes=//g;
              $opackets  =~ s/opackets=//g;
              $obytes    =~ s/obytes=//g;

              if ( $ipackets =~ /M$/ ) {
                $ipackets =~ s/M//g;
                $ipackets = $ipackets * 1000;
              }
              elsif ( $ipackets =~ /G$/ ) {
                $ipackets =~ s/G//g;
                $ipackets = $ipackets * 1000 * 1000;
              }

              if ( $opackets =~ /M$/ ) {
                $opackets =~ s/M//g;
                $opackets = $opackets * 1000;
              }
              elsif ( $opackets =~ /G$/ ) {
                $opackets =~ s/G//g;
                $opackets = $opackets * 1000 * 1000;
              }

              if ( $rbytes =~ /M$/ ) {
                $rbytes =~ s/M//g;
                $rbytes = $rbytes * 1000;
              }
              elsif ( $rbytes =~ /G$/ ) {
                $rbytes =~ s/G//g;
                $rbytes = $rbytes * 1000 * 1000;
              }

              if ( $obytes =~ /M$/ ) {
                $obytes =~ s/M//g;
                $obytes = $obytes * 1000;
              }
              elsif ( $obytes =~ /G$/ ) {
                $obytes =~ s/G//g;
                $obytes = $obytes * 1000 * 1000;
              }
              $ipackets =~ s/K|R//g;
              $opackets =~ s/K|R//g;
              $rbytes   =~ s/K|R//g;
              $obytes   =~ s/K|R//g;
              foreach my $ldom_uuid ( keys %hash_net ) {
                foreach my $net_name ( keys %{ $hash_net{$ldom_uuid} } ) {
                  foreach my $values ( keys %{ $hash_net{$ldom_uuid}{$net_name} } ) {
                    my ( $i_packets, $r_bytes, $o_packets, $o_bytes ) = split( /,/, $hash_net{$ldom_uuid}{$net_name}{$values} );
                    if ( $net_name eq $link_name && $ldom_uuid eq $uuid_ldom ) {
                      my $time_res     = $timestamp_after1 - $timestamp_before1;
                      my $ipackets_res = $ipackets - $i_packets;
                      my $rbytes_res   = $rbytes - $r_bytes;
                      my $opackets_res = $opackets - $o_packets;
                      my $obytes_res   = $obytes - $o_bytes;
                      $ipackets_res = $ipackets_res / $time_res;
                      $rbytes_res   = $rbytes_res / $time_res;
                      $opackets_res = $opackets_res / $time_res;
                      $obytes_res   = $obytes_res / $time_res;
                      $ipackets_res = sprintf "%d", $ipackets_res;
                      $opackets_res = sprintf "%d", $opackets_res;
                      $rbytes_res   = sprintf "%d", $rbytes_res;
                      $obytes_res   = sprintf "%d", $obytes_res;
                      my $ifconfig_output =  `$IFCONFIG`;
                      my $ip_address = "-";
                      if ($ifconfig_output =~ /$link_name:.*?inet\s+(\d+\.\d+\.\d+\.\d+)/s) {
                        $ip_address = $1;
                      }
                      $message .= ":netstat:$uuid_ldom$char_l$ldom_hostname_id$char_l$ip_address:$link_name:$ipackets_res:$rbytes_res:$opackets_res:$obytes_res:\:";
                    }

                  }
                }
              }
            }
          }
        }
      }
    }
  }

  if ( $os =~ m/SunOS/ ) {
    my @smbios           = "";
    my $vm_name          = `uname -n 2>/dev/null`;
    my $test_cmd_netstat  = "ldm list-netstat -p";

    #my @list_of_zone_uuid = "";
    if ( process_limits( "$ZONEADM_P", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$ZONEADM_P\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      @list_of_zone_uuid = `$ZONEADM_P 2>>$error_log`;
    }
    chomp $vm_name;
    my @ldom_info1 = "";
    if ( $type_of_solaris =~ /sparc/ ) {
      if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        @ldom_info1 = `$VIRTINFO 2>>$error_log`;
      }
    }
    my $ldom_stat = "";
    if ( $role_of_ldom =~ /LDoms control/ ) {
      if ( process_limits( "$LDMLS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
        error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLS\", skipping it " . __FILE__ . ":" . __LINE__ );
      }
      else {
        $ldom_stat = `$LDMLS 2>>$error_log`;
      }
    }
    my ( $ldom_name, $status, $vcpu, $all_memory, $util, $uptime, $norm_util, $zone_name, $zone_grep ) = "";
    foreach my $domain ( split( /\n/, $ldom_stat ) ) {
      if ( $role_of_ldom =~ /LDoms control/ ) {
        if ( $domain =~ /DOMAIN|name=/ ) {
          ( undef, $ldom_name ) = split( /\|/, $domain );
          $ldom_name =~ s/name=//g;
          chomp($ldom_name);
          my @ldom_info2 = `$LDMINFO $ldom_name 2>>$error_log`;
          my ($uuid_ldom) = grep /uuid=/, @ldom_info2;
          $uuid_ldom =~ s/UUID\|uuid=//g;
          chomp $uuid_ldom;
          my $ldom_string     = "_ldom";
          my $timestamp_after = time();
          my $net_stat        = "";

          if ( process_limits( "$LDMLSNET", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
            error( "there is already running $PROC_RUN_LIMIT copies of \"$LDMLSNET\", skipping it " . __FILE__ . ":" . __LINE__ );
          }
          else {
            $net_stat = `$LDMLSNET $ldom_name`;
          }
          foreach my $net_ldom ( split /\n/, $net_stat ) {
            if ( $net_ldom =~ /link/ ) {
              my ( $link_name, $ipackets, $rbytes, $opackets, $obytes ) = split( /\|/, $net_ldom );
              $link_name =~ s/link=//g;
              $ipackets  =~ s/ipackets=//g;
              $rbytes    =~ s/rbytes=//g;
              $opackets  =~ s/opackets=//g;
              $obytes    =~ s/obytes=//g;
              if ( $ipackets =~ /M$/ ) {
                $ipackets =~ s/M//g;
                $ipackets = $ipackets * 1000;
              }
              elsif ( $ipackets =~ /G$/ ) {
                $ipackets =~ s/G//g;
                $ipackets = $ipackets * 1000 * 1000;
              }

              if ( $opackets =~ /M$/ ) {
                $opackets =~ s/M//g;
                $opackets = $opackets * 1000;
              }
              elsif ( $opackets =~ /G$/ ) {
                $opackets =~ s/G//g;
                $opackets = $opackets * 1000 * 1000;
              }

              if ( $rbytes =~ /M$/ ) {
                $rbytes =~ s/M//g;
                $rbytes = $rbytes * 1000;
              }
              elsif ( $rbytes =~ /G$/ ) {
                $rbytes =~ s/G//g;
                $rbytes = $rbytes * 1000 * 1000;
              }

              if ( $obytes =~ /M$/ ) {
                $obytes =~ s/M//g;
                $obytes = $obytes * 1000;
              }
              elsif ( $obytes =~ /G$/ ) {
                $obytes =~ s/G//g;
                $obytes = $obytes * 1000 * 1000;
              }
              $ipackets =~ s/K|R//g;
              $opackets =~ s/K|R//g;
              $rbytes   =~ s/K|R//g;
              $obytes   =~ s/K|R//g;
              foreach my $ldom_uuid ( keys %hash_net1 ) {
                foreach my $net_name ( keys %{ $hash_net1{$ldom_uuid} } ) {
                  foreach my $values ( keys %{ $hash_net1{$ldom_uuid}{$net_name} } ) {
                    my ( $i_packets, $r_bytes, $o_packets, $o_bytes ) = split( /,/, $hash_net1{$ldom_uuid}{$net_name}{$values} );
                    if ( $net_name eq $link_name && $ldom_uuid eq $uuid_ldom ) {
                      my $time_res     = $timestamp_after - $timestamp_before;
                      my $ipackets_res = $ipackets - $i_packets;
                      my $rbytes_res   = $rbytes - $r_bytes;
                      my $opackets_res = $opackets - $o_packets;
                      my $obytes_res   = $obytes - $o_bytes;
                      $ipackets_res = $ipackets_res / $time_res;
                      $rbytes_res   = $rbytes_res / $time_res;
                      $opackets_res = $opackets_res / $time_res;
                      $obytes_res   = $obytes_res / $time_res;
                      $ipackets_res = sprintf "%d", $ipackets_res;
                      $opackets_res = sprintf "%d", $opackets_res;
                      $rbytes_res   = sprintf "%d", $rbytes_res;
                      $obytes_res   = sprintf "%d", $obytes_res;
                      $message .= ":vnetstat:$uuid_ldom:$link_name:$ipackets_res:$rbytes_res:$opackets_res:$obytes_res:\:";
                    }

                  }
                }
              }
            }
          }
        }
      }
    }
  }

  #
  #
  # CPU queue

  $message .= cpu_queue( \@vmstat, $os, $wpar_id );

  #
  # DISK ID VOLUME
  #
  my $hour_back_time = $act_time_u - 86400;
  my $last_disk_id_t = 0;
  if ( -f $disk_id_tmp_file ) {
    $last_disk_id_t = ( stat("$disk_id_tmp_file") )[9];
  }

  if ( $hour_back_time > $last_disk_id_t ) {
    my $ret = `touch $disk_id_tmp_file`;       # once a day
    if ($ret) {
      error( "$ret" . __FILE__ . ":" . __LINE__ );
    }
    open( DISKID, "> $disk_id_tmp_file" ) || error( "Cannot open $disk_id_tmp_file: $!" . __FILE__ . ":" . __LINE__ );
    if ( $os =~ /Linux|LINUX/ ) {
      my @volumeid = `$LSDISK 2>/dev/null`;    # there is not everywhere /dev/disk/by-id/wwn*
      print "$LSDISK: Linux disk ID searching ";
      if (@volumeid && defined $volumeid[0] && $volumeid[0] ne "") {
        print "found some ids\n";
      } else {
        print "not found\n";
      }
      foreach my $volume_path (@volumeid) {
        chomp($volume_path);
        $volume_path =~ s/\s+//g;
        $volume_path =~ s/^.*:\d\d//g;
        my ( $id, $disk_name ) = split( /->/, $volume_path );
        $id        =~ s/^.*wwn-//g;
        $id        =~ s/\dx//g;
        $disk_name =~ s/..\/..\///g;
        if ( $id =~ /part/ ) {next}
        $message .= ":disk_id:$disk_name:$id\::::::";
        print "Disk: $disk_name:$id\n";
        print DISKID "$disk_name:$id\n";
      }
    }
    if ( $os =~ /AIX|Aix/ ) {
      if ( -f "/usr/sbin/upadm" ) {
        if ( ! -f "/usr/bin/sudo" ) {
          print STDERR "AIX Huawei disks were detected but no /usr/bin/sudo\n";
          print "AIX Huawei disks were detected but no /usr/bin/sudo\n";
        }
        # Huawei disks
        my @vlun_all = `$LSDISK_AIX_HUAWEI 2>/dev/null`;    # ignore errors
        print "$LSDISK_AIX_HUAWEI: AIX disk ID searching - Huawei\n";
        foreach my $line (@vlun_all) {
          if ( $line =~ /^---|Vlun ID/ ) {next}
          $line =~ s/\s+/ /g;
          my ($disk_name) = ( split( / /, $line ) )[3];
          my ($vlun_wwn)  = ( split( / /, $line ) )[5];
          my ($vlun_name) = ( split( / /, $line ) )[4];

          #$vlun_wwn = substr $vlun_wwn, -8;
          $vlun_wwn  =~ s/://g;
          $vlun_name =~ s/://g;
          $disk_name =~ s/://g;
          $message .= ":disk_id:$disk_name:$vlun_wwn:$vlun_name\:::::";
          print "Disk: $disk_name:$vlun_wwn:$vlun_name\n";
          print DISKID "$disk_name:$vlun_wwn:$vlun_name\n";
        }
      }
      my @volumeid = `$LSDISK_AIX 2>/dev/null`;    # there is not everywhere /dev/disk/by-id/wwn*
      print "$LSDISK_AIX: AIX disk ID searching\n";
      if ( $volumeid[0] eq "" ) {
        print "not found\n";
      }
      else {
        print "found some ids\n";
      }
      foreach my $volume_path (@volumeid) {
        chomp $volume_path;
        my ($disk_name) = split( / /, $volume_path );
        chomp $disk_name;
        if ( $volume_path =~ /SAS Disk Drive|Virtual SCSI/ ) {
          print "Skipping $disk_name as SAS Disk Drive|Virtual SCSI\n";
          next;
        }
        my ($uuid,$name_server)  = get_uuid($disk_name);

        # get disk size from disk
        my $test_disk_size = `getconf DISK_SIZE /dev/$disk_name 2>>$error_log`;
        my $disk_size = "";
        if ($test_disk_size){
          chomp $test_disk_size;
          if ( isdigit($test_disk_size) ) {
            $disk_size = $test_disk_size;
          }
        }
        $name_server = "" unless defined $name_server;
        $uuid        =~ s/://g if defined $uuid;
        $name_server =~ s/://g if defined $name_server;
        $disk_name   =~ s/://g if defined $disk_name;
        $message .= ":disk_id:$disk_name:$uuid:$name_server:$disk_size\::::";
        print "Disk: $disk_name:$uuid:$name_server\n";
        print DISKID "$disk_name:$uuid:$name_server\n";
      }
    }
    if ( $os =~ /SunOS/ ) {
      my @iostat_name = `$IOSTAT_SOL 2>>$error_log`;
      print "$IOSTAT_SOL: Solaris disk ID searching";

      if ( $iostat_name[0] eq "" ) {
        print "not found\n";
      } else {
        print "found some ids\n";
      }
      my @prtconf_all = `prtconf -v 2>/dev/null`;
      my %disk_wwn_map;
      my $current_guid;
      my $in_disk_block = 0;
      foreach my $line (@prtconf_all) {
        # start of new disk block
        if ( $line =~ /^\s*disk,\s+instance/ ) {
          $in_disk_block = 1;
          undef $current_guid;
          next;
        }
        next unless $in_disk_block;
        # find WWN
        if ( $line =~ /name='client-guid'/ ) {
          next;    # value is another row
        }
        if ( $line =~ /value='([0-9a-f]+)'/i && !defined $current_guid ) {
          $current_guid = lc $1;
          next;
        }
        # find mapping to /dev/dsk
        if ( $line =~ m{/dev/dsk/(c\d+t[0-9A-F]+d\d+)s\d}i ) {
          my $disk = lc $1;
          $disk_wwn_map{$disk} = uc $current_guid if $current_guid;
        }
      }
      foreach my $iostat_line (@iostat_name) {
        if ( $iostat_line =~ /c\d/ ) {
          my ($disk_name) = split( / /, $iostat_line );
          $disk_name =~ s/://g;
          $disk_name = lc $disk_name;

          if ( my $id = $disk_wwn_map{$disk_name} ) {

            $message .= ":disk_id:$disk_name:$id\::::::";
            print "Disk: $disk_name:$id\n";
            print DISKID "$disk_name:$id\n";
          }
          else {
            print "Disk: $disk_name — WWN not found in prtconf\n";
          }
        }
      }
    }
    close(DISKID);
  }

  # started already on the top
  if ( defined $ps_job_output ) {
    $message .= $ps_job_output;
  }

  # if ($model !~ /Solaris|Solaris10|Solaris11/){
  #
  # KVM
  #
  #   my $tool_name = "virsh";
  #   my $tool_path = `which $tool_name 2>/dev/null`;
  #   if ( $tool_path ) {
  #     $message .= ":" . kvm_stats();
  #   }
  # }

  if ( $os eq "Linux" ) {
    my @CPU_ARRAY = `$CPU_LINUX`;
    my ($cpu_count)        = grep {/CPU\(s\)/} @CPU_ARRAY;
    my ($cpu_mhz)          = grep {/CPU MHz:/} @CPU_ARRAY;
    my ($threads_per_core) = grep {/Thread\(s\) per core:/} @CPU_ARRAY;
    my ($cores_per_socket) = grep {/Core\(s\) per socket/} @CPU_ARRAY;
    my ($model_name)       = grep {/Model name:/} @CPU_ARRAY;
    my ($architecture)     = grep {/Architecture:/} @CPU_ARRAY;
    my ($sockets)          = grep {/Socket\(s\):/} @CPU_ARRAY;
    my ($hyper_vendor)     = grep {/Hypervisor vendor:/} @CPU_ARRAY;
    # RHEL9 Linux does not provide info about CPU MHz in seperate lscpu
    if (!$cpu_mhz){
      my $cpu_info_about_mhz = `lscpu -e=MHZ 2>/dev/null`;
      my @lines = split /\n/, $cpu_info_about_mhz;
      $cpu_mhz  = $lines[1];
    }
    for ($cpu_count, $cpu_mhz, $threads_per_core, $cores_per_socket, $model_name, $architecture, $sockets, $hyper_vendor) {
      chomp if defined $_;
      next unless defined $_;
      s/^\s*.*?:\s*//;
    }
    for ($cpu_count, $cpu_mhz, $threads_per_core, $cores_per_socket, $model_name, $architecture, $sockets, $hyper_vendor) {
      $_ = '' unless defined $_;
    }
    $cpu_count        = defined $cpu_count        ? $cpu_count        : '';
    $threads_per_core = defined $threads_per_core ? $threads_per_core : '';
    $cores_per_socket = defined $cores_per_socket ? $cores_per_socket : '';
    $model_name       = defined $model_name       ? $model_name       : '';
    $architecture     = defined $architecture     ? $architecture     : '';
    $sockets          = defined $sockets          ? $sockets          : '';
    $hyper_vendor     = defined $hyper_vendor     ? $hyper_vendor     : '';
    if (!defined $cpu_mhz || $cpu_mhz !~ /\d/) {
      $cpu_mhz = "unknown";
    }
    # get linux os
    my $linux_distro = "";
    if ( -e "/etc/system-release" ) {
      if ( open my $fh, '<', '/etc/system-release' ) {
        chomp( $linux_distro = <$fh> );
        close $fh;
      }
    }
    elsif ( -e "/etc/os-release" ) {
      if ( open my $fh, '<', '/etc/os-release' ) {
        while ( my $line = <$fh> ) {
          if ( $line =~ /^PRETTY_NAME=(.*)/ ) {
            my $pretty = $1;
            $pretty =~ s/^"//; 
            $pretty =~ s/"$//;
            chomp $pretty;
            $linux_distro = $pretty;
            last;
          }
        }
        close $fh;
      }
    }
    if ( $cpu_count && $cpu_mhz =~ /\d/ ) {
      $message .= ":linux_cpu:$cpu_count:$cpu_mhz:$threads_per_core:$cores_per_socket:$model_name:$architecture:$sockets:$hyper_vendor";
    }
    # only for NG conf
    $message .= ":linux_info_cpu:$cpu_count:$cpu_mhz:$threads_per_core:$cores_per_socket:$model_name\/$linux_distro:$architecture:$sockets:$hyper_vendor";
  }
  #
  #
  # MULTIPATH Linux
  # compellent-demo02 (36000d3100377ce000000000000000011) dm-3 COMPELNT,Compellent Vol
  # size=100G features='1 queue_if_no_path' hwhandler='0' wp=rw
  # -+- policy='service-time 0' prio=1 status=active
  #  |- 1:0:0:2 sdb 8:16 active ready running
  #  - 2:0:1:2 sde 8:64 active ready running
  # compellent-demo (36000d3100377ce00000000000000000d) dm-2 COMPELNT,Compellent Vol
  # size=100G features='1 queue_if_no_path' hwhandler='0' wp=rw
  # -+- policy='service-time 0' prio=1 status=active
  #  |- 1:0:1:1 sdc 8:32 active ready running
  #  - 2:0:0:1 sdd 8:48 active ready running
  # LIST_DISK - command "lsblk -d"
  # NAME MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
  # sda    8:0    0   30G  0 disk
  # sdb    8:16   0  100G  0 disk
  # sdc    8:32   0  100G  0 disk
  # sdd    8:48   0  100G  0 disk
  # sde    8:64   0  100G  0 disk
  #
  if ( $os eq "Linux" ) {
    my $hour_back_time = $act_time_u - 3600;
    my $last_multi_t   = 0;
    if ( -f $multipath_tmp_file ) {
      $last_multi_t = ( stat("$multipath_tmp_file") )[9];
    }
    if ( $hour_back_time > $last_multi_t ) {
      my $ret = `touch $multipath_tmp_file`;    # once a day
      $message .= multipath_linux( $MULTI_LIN, $os );
      my @exclude_disks = "";
      if ( -f "$multipath_exclude_file" ) { # list of excludes disk
        open( FH, "< $multipath_exclude_file" ) || error( " Can't open $multipath_exclude_file: $! " . __FILE__ . ":" . __LINE__ ) && next;
        my @data_from_file = <FH>;
        close(FH);
        @exclude_disks = grep {!/^\#/} @data_from_file;
      }
      if ( -f $multi_lin_data_file ) {
        open( MULTI_L, "< $multi_lin_data_file" ) || error( "Cannot open $multi_lin_data_file: $!" . __FILE__ . ":" . __LINE__ );
        my @multi_data = <MULTI_L>;
        my @list_disk_cmd     = `$LIST_DISK`;
        #print "@list_disk_cmd\n";
        my %hash_test_multi = ();
        foreach my $line_disk (@list_disk_cmd){
          chomp $line_disk;
          if ($line_disk =~ /^NAME/){next}
          $line_disk =~ s/\s+/,/g;
          my ($disk_name,undef,undef,$capacity_in_GB) = split (/,/,$line_disk);
          my ($grep_disk) = grep (/$disk_name/,@multi_data);
          if ($grep_disk){
            $grep_disk =~ s/=====double-colon=====/:/g;
            my ($disk_name_in_multi) = split (/ /,$grep_disk);
            $hash_test_multi{$disk_name_in_multi} = $capacity_in_GB;
          }
          else{
            $disk_name = "/dev/$disk_name";
            $hash_test_multi{$disk_name} = $capacity_in_GB;
          }
        }
        foreach my $disk_name ( sort keys (%hash_test_multi) ) {
          if ( $disk_name =~ m/^\/dev\/loop/ ) {
            next;
          }
          my $capacity_disk = $hash_test_multi{$disk_name};
          $message .= ":capacity_disk_lin:$disk_name:$capacity_disk:\:::::";
        }

        close(MULTI_L);
        unlink($multi_lin_data_file);
        foreach my $line (@multi_data) {
          chomp $line;
          my ( $alias, $size, $path_group, $path, $path_status ) = split( /:/, $line );
          my $string_with_alias  = "$alias";
          my $alias_for_check = "";
          if ( $string_with_alias =~ /\(/ ) { # name disk with undesirable text
            ( $alias_for_check ) = split( /\(/, $string_with_alias  );
            $alias_for_check =~ s/\s+$//g;
          } else { $alias_for_check = $alias }
          my $grep_disk = grep /^$alias_for_check$/, @exclude_disks;
          if ($grep_disk == 1) {
            print "multipath-exclude.txt file exists: skipping disk $alias_for_check\n";
            next;
          } # if want to skip some disks
          #print "$alias,$size,$path_group,$path\n";
          $message .= ":path_lin:$alias:$size:$path_group:$path:\:::";
        }
      }
    }
  }
  #
  #
  # MULTIPATH AIX
  if ( $os eq "AIX" ) {
    my $hour_back_time = $act_time_u - 3600;
    my $last_multi_t   = 0;
    if ( -f $multipath_tmp_file ) {
      $last_multi_t = ( stat("$multipath_tmp_file") )[9];
    }

    if ( $hour_back_time > $last_multi_t ) {
      my $ret = `touch $multipath_tmp_file`;    # once a day
      $message .= multipath_aix( $LSPATH, $os );

      if ( -f $lsstat_data_file ) {
        open( LS, "< $lsstat_data_file" ) || error( "Cannot open $lsstat_data_file: $!" . __FILE__ . ":" . __LINE__ );
        my @ls_data = <LS>;
        close(LS);
        unlink($lsstat_data_file);
        foreach my $line (@ls_data) {
          chomp $line;
          my ( $name, $path_id, $connection, $parent, $path_status, $status, $final_status ) = split( /:/, $line );
          # get disk size from disk
          my $test_disk_size = `getconf DISK_SIZE /dev/$name 2>/dev/null`;
          my $disk_size = "";
          if ($test_disk_size){
            chomp $test_disk_size;
            if ( isdigit($test_disk_size) ) {
              $disk_size = $test_disk_size;
            }
          }
          $message .= ":lsdisk:$name:$path_id:$connection:$parent:$path_status:$status:$disk_size:$final_status";
        }
      }
    }
  }
  #
  #
  # MULTIPATH Solaris
  if ( $os eq "SunOS" ) {
    my $hour_back_time = $act_time_u - 3600;
    my $last_multi_t   = 0;
    if ( -f $multipath_tmp_file ) {
      $last_multi_t = ( stat("$multipath_tmp_file") )[9];
    }

    if ( $hour_back_time > $last_multi_t ) {
      my $ret = `touch $multipath_tmp_file`;    # once a day
      $message .= multipath_solaris( $MULTI_SOL, $os );

      if ( -f $multi_sol_data_file ) {
        open( MULTI_S, "< $multi_sol_data_file" ) || error( "Cannot open $multi_sol_data_file: $!" . __FILE__ . ":" . __LINE__ );
        my @solaris_data = <MULTI_S>;
        close(MULTI_S);
        unlink($multi_sol_data_file);
        foreach my $line (@solaris_data) {
          chomp $line;
          my ( $disk_id, $disk_alias, $vendor, $product, $revision, $name_type, $asymmetric, $curr_load_balance, $string1, $string2, $string3, $string4 ) = split( /:/, $line );
          $message .= ":path_sol:$disk_id/$disk_alias/$vendor/$product/$revision/$name_type/$asymmetric/$curr_load_balance:$string1:$string2:$string3:$string4:\::";
        }
      }
    }
  }
  #
  # FS monitoring
  #
  if ( -e $fs_storae_file ) {
    my $FS_timestamp = ( stat($fs_storae_file) )[9];
    my $time_diff    = $act_time_u - $FS_timestamp;
    if ( $time_diff > 480 ) { #must be 10m -2 minutes, because of alerting on NG
      $message .= translate_df_output( $DF_AIX7, $DF_AIX, $DF_SOL, $DF_LIN, $os );
    }
  }
  else {
    my $temp = translate_df_output( $DF_AIX7, $DF_AIX, $DF_SOL, $DF_LIN, $os );
    $message .= $temp;
  }
  # !!! XORMON-NG NEW DATA MUST BE HERE!!! LPAR2RRD not supported items !!!!
  if ( $os =~ m/SunOS/ ) {
    # the netstat command does not work, we will find all IP addresses.
    if (system("command -v ldm >/dev/null 2>&1") != 0) {
      my $ifconfig_output =  `$IFCONFIG`;
      my @lines = split /\n/, $ifconfig_output;
      my $interface;
      foreach my $line (@lines) {
        chomp $line;
        if ($line =~ /^(\S+):\s/) {
          $interface = $1;
        }
        if (defined $interface && $line =~ /\binet\s([\d\.]+)\s.*broadcast\s([\d\.]+)/) {
          my ($ip, $broadcast) = ($1, $2);
          $interface =~ s/:/-/g;    # replace : by "-" cause ":" is a problem for parsing
          $ip =~ s/:/./g;
          $message .= ":ip_net:$interface:$ip\::::::";
        }
      }
    } else {
      my $ifconfig_output =  `$IFCONFIG`;
      my @lines = split /\n/, $ifconfig_output;
      my $interface;
      foreach my $line (@lines) {
        chomp $line;
        if ($line =~ /^(\S+):\s/) {
          $interface = $1;
        }
        if (defined $interface && $line =~ /\binet\s([\d\.]+)\s.*broadcast\s([\d\.]+)/) {
          my ($ip, $broadcast) = ($1, $2);
          $interface =~ s/:/-/g;    # replace : by "-" cause ":" is a problem for parsing
          $ip =~ s/:/./g;
          $message .= ":ip_net:$interface:$ip\::::::";
        }
      } 
      #}
    }
  }
  #
  # NFS IOSTAT
  # 
  if ( $os =~ /Linux/ ) {
    if ( defined $pid_nfs_iostat ) {
      waitpid( $pid_nfs_iostat, 0 );
    }
    if ( -f $nfs_iostat_data_file ) {
      open( IO, "< $nfs_iostat_data_file" ) || error( "Cannot open $iostat_data_file: $!" . __FILE__ . ":" . __LINE__ );
      my @nfs_iostat_data = <IO>;
      close(IO);
      unlink($nfs_iostat_data_file);
      foreach my $line (@nfs_iostat_data) {
        chomp $line;
        my @fields = split /,/, $line;

        # Mount point always first
        my $mount = shift @fields;

        my $half = int(@fields / 2);
        my @read_vals  = @fields[0 .. $half - 1];
        my @write_vals = @fields[$half .. $#fields];

        push @read_vals,  ('-') x (6 - @read_vals)  if @read_vals  < 6;
        push @write_vals, ('-') x (6 - @write_vals) if @write_vals < 6;

        # undef to "-"
        foreach (@read_vals, @write_vals) {
          $_ = "-" if !defined($_) || $_ eq '';
        }

        $message .= ":nfs_iostat_read:$mount:" . join(":", @read_vals) . ":";
        $message .= ":nfs_iostat_write:$mount:" . join(":", @write_vals) . ":";
      }
    }
  }

  # ZFS monitoring save data to message
  if ($os =~ /Linux/ && defined $pid_zfs_iostat) {
    waitpid( $pid_zfs_iostat, 0 );
    if ( -f $zfs_iostat_data_file ) {
      open( IO, "< $zfs_iostat_data_file" ) || error( "Cannot open $zfs_iostat_data_file: $!" . __FILE__ . ":" . __LINE__ );
      my @zfs_iostat_data = <IO>;
      close(IO);
      my $zfs_message = join("", @zfs_iostat_data);
      chomp($zfs_message);
      $message .= $zfs_message;
      unlink($zfs_iostat_data_file);
    }
  }

  # Volumes group - filesystem
  if ($os eq 'AIX'){
    my $six_half_hour_back_time = $act_time_u - 23400;
    my $last_volumes_disk_t = 0;
    if ( -f $volume_list_tmp_file ) {
      $last_volumes_disk_t = ( stat("$volume_list_tmp_file") )[9];
    }
    if ( $six_half_hour_back_time > $last_volumes_disk_t ) {
      my $ret = `touch $volume_list_tmp_file`; # once per 6.5 hours
      my $list_of_volumes = `lspv 2>/dev/null`;
      my $mount_fs        = `mount 2>/dev/null`;
      my %vg_data;
      my %mounted_fs;
      for my $line (split /\n/, $mount_fs) {
        next if $line =~ /^\s*node|^\s*-{3}/;
        my @cols = split /\s+/, $line;
        next unless @cols >= 3;
        my $mount_point = $cols[2];
        $mounted_fs{$mount_point} = 1;
      }
      #print Dumper \%mounted_fs;
      for my $line (split /\n/, $list_of_volumes) {
        chomp $line;

        my ($disk_name,$pvid,$vg,$status) = split(/\s+/, $line);
        if ($status ne "active" && $status ne "concurrent") { next }

        my $lun_id = get_uuid($disk_name);
        my $vg_list_of_fs = `lsvgfs $vg 2>/dev/null`;
        my @filesystems = split(/\n/, $vg_list_of_fs);
        my @mounted_filesystems = grep { exists $mounted_fs{$_} } @filesystems; # only FS in mount
        my $fs_joined = @mounted_filesystems ? join(";", @mounted_filesystems) : '';
        if (exists $vg_data{$vg}) {
          $vg_data{$vg}->{disks} .= ";$disk_name";
          $vg_data{$vg}->{pvids} .= ";$pvid";
        } else {
          $vg_data{$vg} = {
            fs     => $fs_joined,
            disks  => $disk_name,
            lun_id  => $lun_id
          };
        }
        #print Dumper \%vg_data;
        if (%vg_data) {
          for my $vg (keys %vg_data) {
            #print "volumes_list:$vg:$vg_data{$vg}->{fs}:$vg_data{$vg}->{lun_id}:$vg_data{$vg}->{disks}:\:::\n";
            $message .= ":volumes_list:$vg:$vg_data{$vg}->{fs}:$vg_data{$vg}->{lun_id}:$vg_data{$vg}->{disks}:\:::";
          }
        }
      }
    }
    # AIX FCP/FCA error log
    if ( defined $ENV{DEBUG} ) {
      $debug_aix_error = $ENV{DEBUG};
    }
    my $fifteen_back_time  = $act_time_u - 900;
    my $last_aixerr_time   = 0;
    if ( -f $err_log_tmp_file ) {
      $last_aixerr_time = ( stat("$err_log_tmp_file") )[9];
    }
    if ( !$err_log_tmp_file || $fifteen_back_time > $last_aixerr_time ) {
      my $ret = `touch $err_log_tmp_file`;
      my $lines = aixerrpt();
      if ($lines){
        $message .= "$lines";
        #print "$lines\n";
      }
    }
  }

  # info about adapters
  if ( $os =~ /AIX|Aix/ ) {
    my $day_back_time = $act_time_u - 86400;
    my $last_eth_t     = 0;
    if ( -f $eth_hw_tmp_file ) {
      $last_eth_t = ( stat($eth_hw_tmp_file) )[9];
    }
    if ( $day_back_time > $last_eth_t ) {
      my $ret = `touch $eth_hw_tmp_file`;    # once a day
      my @output_adapters = `$lscfg_adapters 2>/dev/null`;
      my @pairs;
      for my $line (@output_adapters) {
        chomp $line;
        # only ent[0-9]
        next unless $line =~ /\bent\d+\b/;

        my @fields = split ' ', $line;
        my $ent  = $fields[1];
        my $hwid = $fields[2];
        push @pairs, "$ent,$hwid";
      }
      $message .= ":eth-hw-path:" . join(",", @pairs) . ":\::::::";
    }
  }

  #
  # LINUX syslog dmesg
  #
  # Example:
  # [root@vm-jirik lpar2rrd]# LANG=C dmesg -T --ctime --level=alert,crit,err,warn
  # [Thu Sep  1 20:23:32 2022] ACPI: RSDP 00000000000f6a00 00024 (v02 PTLTD )
  # [Thu Sep  1 20:23:32 2022] ACPI: FACP 00000000bfefee73 000F4 (v04 INTEL  440BX    06040000 PTL  000F4240)
  # [Thu Sep  1 20:23:42 2022] raid6: using algorithm avx2x4 gen() (10273 MB/s)
  # [Thu Sep  1 20:23:43 2022] piix4_smbus 0000:00:07.3: Host SMBus controller not enabled!
  # [Thu Sep  1 20:23:43 2022] intel_rapl: no valid rapl domains found in package 0
  # [Wed Oct  5 10:25:59 2022] hrtimer: interrupt took 13961084 ns
  if ($os eq 'Linux'){
    # last time from stamp file
    my $first_run = 0;
    my $last_ts = 0;
    my $hour_back_time = time() - 3600;
    my $last_run_time  = 0;

    if (-f $linux_syslog_tmp_file) {
      $last_run_time = (stat($linux_syslog_tmp_file))[9];
    }

    if ($hour_back_time > $last_run_time) {
      my $ret = `touch $linux_syslog_tmp_file`;
      # loading the previous timestamp (last dmesg)
      if (-f $linux_syslog_file) {
        open my $fh, '<', $linux_syslog_file;
        my $line = <$fh>;
        chomp $line if $line;
        $last_ts = $line if $line =~ /^\d+$/;
        close $fh;
      } else {
        $first_run = 1;
      }

      my @lines;
      my @levels = qw(alert crit err warn);
      foreach my $level (@levels) {
        my $cmd = "$syslog_cmd --level=$level";
        my @dmesg_output = `$cmd 2>/dev/null`;
        foreach my $line (@dmesg_output) {
          chomp $line;
          if ($line =~ /^\[([A-Z][a-z]{2}) ([A-Z][a-z]{2})\s+(\d+)\s+(\d+):(\d+):(\d+)\s+(\d{4})\] (.+)$/) {
            my ($dow, $mon_str, $mday, $hour, $min, $sec, $year, $msg) = ($1,$2,$3,$4,$5,$6,$7,$8);
            my $mon = $month_map{$mon_str};
            next unless defined $mon;
            my $epoch = mktime($sec, $min, $hour, $mday, $mon, $year - 1900);
            next if $epoch <= $last_ts;
            push @lines, [$epoch, $epoch, $msg, $level];
          }
        }
      }
      # First run -> only save last timestamp
      if ($first_run) {
        if (@lines) {
          my $latest = (sort { $b->[0] <=> $a->[0] } @lines)[0]->[0];
          open my $fh, '>', $linux_syslog_file or warn "Cannot write $linux_syslog_file: $!";
          print $fh "$latest\n";
          close $fh;
        }
        print "Linux syslog: dmesg first run\n";
      }
      else{
        # Save new messages and save last timestamp
        if (@lines) {
          my $max_ts = 0;
          foreach my $rec (@lines) {
            my ($ts, $ts_str, $msg, $level) = @$rec;
            #$ts_str =~ s/:/=====double-colon=====/g;
            $msg    =~ s/:/=====double-colon=====/g;
            $message .= ":dmesg:${ts}:${msg}:${level}::\:::";
            $max_ts = $ts if $ts > $max_ts;
          }
          open my $fh, '>', $linux_syslog_file or warn "Cannot write $linux_syslog_file: $!";
          print $fh "$max_ts\n";
          close $fh;
        }
      }
    } else {
      print "Linux syslog: skipping dmesg run - last run was less than an hour ago\n";
    }
  }
  #
  # Linux LVM
  #
  if ($os eq 'Linux'){
    my $LSBLK  = "/bin/lsblk -P -o NAME,KNAME,TYPE,MOUNTPOINT";
    my @lines  = `$LSBLK 2>> $error_log`;
    chomp @lines;
    my $six_half_hour_back_time = $act_time_u - 23400;
    my $last_volumes_disk_t = 0;
    if ( -f $volume_list_tmp_file ) {
      $last_volumes_disk_t = ( stat("$volume_list_tmp_file") )[9];
    }
    if ( $six_half_hour_back_time > $last_volumes_disk_t ) {
      my $ret = `touch $volume_list_tmp_file`; # once per 6.5 hours

      if (! -x "/bin/lsblk") {
        print "cmd /bin/lsblk does not exist or is not executable!\n";
      } else {
        my @lines  = `$LSBLK 2>> $error_log`;
        chomp @lines;

        if (!@lines) {
          print "cmd '$LSBLK' did not return any output!\n";
        }
        else {
          my %seen;
          foreach my $line (@lines) {
            my %h;
            while ($line =~ /(\w+)="(.*?)"/g) {
              $h{$1} = $2;
            }
            next unless $h{TYPE} eq "lvm";

            my $dm  = $h{KNAME};
            my $mnt = $h{MOUNTPOINT} || "-";

            my $sysname = `cat /sys/block/$dm/dm/name 2>/dev/null`;
            chomp $sysname;
            my ($vg, $lv) = parse_vg_lv($sysname);

            my ($parents, $wwns) = resolve_parents($dm);

            for my $i (0 .. $#$parents) {
              my $parent = $parents->[$i];
              my $wwn    = $wwns->[$i];
              my $key    = "$vg|$lv|$mnt|$parent|$wwn";

              next if $seen{$key}++;
              $message .= ":linux_lvm:$vg:$lv:$mnt:$wwn:$parent:\::";
              #print "$vg,$lv,$mnt,$wwn,$parent\n";
            }
          }
        }
      }
    }
  }

  my ( $some_psi_cpu_avg, $full_psi_cpu_avg, $some_psi_io_avg, $full_psi_io_avg, $some_psi_mem_avg, $full_psi_mem_avg ) = "";

  my $psi_found = 0;

  # CPU PSI
  if ( -f $psi_cpu ) {
    if ( open my $fh, '<', $psi_cpu ) {
      while ( my $line = <$fh> ) {
        chomp $line;

        if ( $line =~ /^some/ && $line =~ /avg60=([\d\.]+)/ ) {
          $some_psi_cpu_avg = $1;
          $psi_found = 1;
        }

        if ( $line =~ /^full/ && $line =~ /avg60=([\d\.]+)/ ) {
          $full_psi_cpu_avg = $1;
          $psi_found = 1;
        }
      }
      close $fh;
    }
  }

  # IO PSI
  if ( -f $psi_io ) {
    if ( open my $fh, '<', $psi_io ) {
      while ( my $line = <$fh> ) {
        chomp $line;

        if ( $line =~ /^some/ && $line =~ /avg60=([\d\.]+)/ ) {
          $some_psi_io_avg = $1;
          $psi_found = 1;
        }

        if ( $line =~ /^full/ && $line =~ /avg60=([\d\.]+)/ ) {
          $full_psi_io_avg = $1;
          $psi_found = 1;
        }
      }
      close $fh;
    }
  }

  # MEMORY PSI
  if ( -f $psi_mem ) {
    if ( open my $fh, '<', $psi_mem ) {
      while ( my $line = <$fh> ) {
        chomp $line;

        if ( $line =~ /^some/ && $line =~ /avg60=([\d\.]+)/ ) {
          $some_psi_mem_avg = $1;
          $psi_found = 1;
        }

        if ( $line =~ /^full/ && $line =~ /avg60=([\d\.]+)/ ) {
          $full_psi_mem_avg = $1;
          $psi_found = 1;
        }
      }
      close $fh;
    }
  }

  if ($psi_found) {
    $message .= ":psi:$full_psi_cpu_avg:$some_psi_cpu_avg:$full_psi_io_avg:$some_psi_io_avg:$full_psi_mem_avg:$some_psi_mem_avg:\:";
  }


  if ($cloud_uuid){
    $message .= ":system_info:$cloud_uuid:\::::::";
  }
  print "$message\n";
  return $message;
}

sub aixerrpt {
  #
  my ($data, $identifier, $timestamp, $resource, $description, @error, @ignore);
  #
  my $conf_file = "$inputdir_aix" . "/aixerrpt.conf";
  my $stat_file = "$base_dir/lpar2rrd-agent-$host-$user_name-aixerrpt.stat";
  my $errpt_fh;
  my $all_output_err_lines;
  if ( -r $conf_file ) {
    open my $cfh, '<', $conf_file or do {
      warn "AIX Error log: cannot open conf file $conf_file\n";
      return 0;
    };
    while (my $line = <$cfh>) {
      chomp($line);
      next if $line =~ /^\s*#/;
      if ($line =~ /"([A-F0-9]+)"/) {
        push @ignore, $1;
      }
    }
    close $cfh;
  } else {
    warn "AIX Error log: conf file $conf_file does not exist or is not readable\n";
    return 0;
  }
  if ( ! -e $stat_file ) {
    open my $fh, ">", $stat_file;
    if ($fh) {
      print $fh "";
      close $fh;
    } else {
      print "Warning: Could not create file $stat_file.\n";
    }
  }
  # add the errpt header line to the ignore list ...
  push (@ignore, "IDENTIFIER");
  print ("aixerrpt: DEBUG: IGNORE:\n", Dumper( \@ignore ), "\n" ) if ( $debug_aix_error );

  my $stat = read_stat( $stat_file );
  print ("aixerrpt: DEBUG: Last errpt status info:\n", Dumper( $stat ), "\n" ) if ( $debug_aix_error );

  my $newstat = new_stat();
  print ("aixerrpt: DEBUG: Last errpt status info:\n", Dumper( $newstat ), "\n" ) if ( $debug_aix_error );
  write_stat( $stat_file, $newstat );
  #print Dumper $stat;
  if (ref $stat eq 'HASH' && defined $stat->{sequence}) {
    print STDOUT ( "aixerrpt (v" . $version . "): starting aixerrpt at " . $stat->{starttime} . " and Sequence number " . $stat->{sequence} . ", using conf file $conf_file.\n" );
  }
  elsif ($stat == 1) { # first run, return 1 and script can move on
    print "AIX Error log: first run of the agent – sequence and timestamp must be written to the auxiliary file – $stat_file.\n";
    return 1;
  }
  else {
    print STDOUT ( "aixerrpt (v" . $version . "): starting aixerrpt at " . $stat->{starttime} . " , using conf file $conf_file.\n" );
  }

  # errpt - error report list from last time
  open ($errpt_fh,'-|', "/usr/bin/errpt -a -s " . $stat->{starttime} ) or warn "aixerrpt: couldn't open errpt for reading!\n";

  # while the above process is open ...
  my $flag_descr = 0;
  my $flag_vpd = 0;
  my $flag_sense = 0;
  my $num_sense = 0;
  my $rec;
  while (my $line = <$errpt_fh>) {
    chomp( $line );
    if( $flag_descr ) {
      if( $line ) {
        $rec->{'Description'} = $line;
      }
      else {
          $flag_descr = 0;
      }
    }
    elsif( $flag_sense ) {
      $num_sense++;
      if( $line =~ /^[-]{10,}$/ ) {
        $flag_sense = 0;
        $num_sense = 0;
      }
      else {
        if( $rec->{'LABEL'} =~ /^FCP_ERR/ ) {
          if( not defined $rec->{'SENSE DATA'}->{'FCP_ERR'} ) {
            $rec->{'SENSE DATA'}->{'FCP_ERR'} = {};
          }
          parse_sense_fcperr( $rec->{'SENSE DATA'}->{'FCP_ERR'}, $line, $num_sense );
        }
        elsif( $rec->{'LABEL'} =~ /^FCA_ERR/ ) {
          if( not defined $rec->{'SENSE DATA'}->{'FCA_ERR'} ) {
            $rec->{'SENSE DATA'}->{'FCA_ERR'} = {};
          }
          parse_sense_fcaerr( $rec->{'SENSE DATA'}->{'FCA_ERR'}, $line, $num_sense );
        }
        push( @{$rec->{'SENSE DATA RAW'}}, $line );
      }
    }
    elsif( $line =~ /^(LABEL):\s+(\w+)$/ ) {
      # new record
      #print( "__DBG__: LINE: $line\n" );
      $rec = {};
      $rec->{$1} = $2;
    }
    elsif( $line =~ /^(Sequence Number):\s+(\w+)$/ ) {
      $rec->{$1} = $2;
      if ( defined $rec->{'Sequence Number'} && $rec->{'Sequence Number'} ) {
        $data->{ $rec->{'Sequence Number'} } = $rec;
      }
    }
    elsif( $line =~ /^(VPD):\s*$/ ) {
      $rec->{'VPD'} = {};
      $flag_vpd = 1;
    }
    elsif( $line =~ /^Description$/ ) {
      $flag_vpd = 0;
      $flag_descr = 1;
    }
    elsif( $line =~ /^SENSE DATA$/ ) {
      $rec->{'SENSE DATA'} = {};
      $rec->{'SENSE DATA RAW'} = [];
      $flag_sense = 1;
    }
    elsif( $line =~ /^Date\/Time:\s+(.*)$/ ) {
      $rec->{'DateTime'} = $1;
    }
    elsif( $line =~ /^([\s\w]+):\s+(.*)$/ ) {
      $rec->{$1} = $2;
    }
    elsif( $flag_vpd && $line =~ /^\s+([\s\w]+)[.]+([.\s\w]+)$/ ) {
      $rec->{'VPD'}->{$1} = $2;
    }
    elsif( $flag_vpd && $line =~ /^\s+Device Specific.\(([\w]+)\)[.]+([.\s\w]+)$/ ) {
      $rec->{'VPD'}->{'Device Specific'}->{$1} = $2;
    }
  }
  close( $errpt_fh );
  #
  # Remove duplicated entry
  if ( defined $stat->{sequence} && $stat->{sequence} ) {
    foreach my $seqnum ( sort {$a <=> $b} ( keys( %{$data} ) ) ) {
      if ( $seqnum <= $stat->{sequence} ) {
        printf ( "aixerrpt: DEBUG: Remove duplicated: %d,%s,%s,%s\n", $data->{$seqnum}->{'Sequence Number'}, $data->{$seqnum}->{'DateTime'}, $data->{$seqnum}->{'IDENTIFIER'}, $data->{$seqnum}->{'Description'} ) if ( $debug_aix_error );
        delete( $data->{$seqnum} );
      }
    }
  }
  #
  #write_log( $log_file, $data );
  #
  #print "==========DATA==============\n";
  #print Dumper ($data);
  #print "==========END DATA===========\n";
  foreach my $seqnum (sort { $a <=> $b } keys %{$data}) {
    my $rec = $data->{$seqnum};

    my $datetime = "";
    if (defined $rec->{'DateTime'}) {
      $datetime = $rec->{'DateTime'};
      $datetime =~ s/:/=====double-colon=====/g;
    }

    my $node_id = "";
    if (defined $rec->{'Node Id'}) {
      $node_id = $rec->{'Node Id'};
    }

    my $resource_name = "";
    if (defined $rec->{'Resource Name'}) {
      $resource_name = $rec->{'Resource Name'};
      $resource_name =~ s/^\s+|\s+$//g;
    }

    my $resource_type = "";
    if (defined $rec->{'Resource Type'}) {
      $resource_type = $rec->{'Resource Type'};
    }

    my $description = "";
    if (defined $rec->{'Description'}) {
      $description = $rec->{'Description'};
      $description =~ s/:/=====double-colon=====/g;
    }

    my $label = "";
    if (defined $rec->{'LABEL'}) {
      $label = $rec->{'LABEL'};
    }

    my $remote_wwnn = "";
    my $remote_wwpn = "";
    my $local_wwpn  = "";
    if (defined $rec->{'SENSE DATA'} && ref($rec->{'SENSE DATA'}) eq 'HASH' && scalar keys %{$rec->{'SENSE DATA'}} > 0) {
      my ($type) = keys %{$rec->{'SENSE DATA'}};
      my $entry = $rec->{'SENSE DATA'}->{$type};
      if (defined $entry) {
        if (defined $entry->{'Remote WWNN'}) {
          $remote_wwnn = $entry->{'Remote WWNN'};
          $remote_wwnn =~ s/:/=====double-colon=====/g;
        }
        if (defined $entry->{'Remote WWPN'}) {
          $remote_wwpn =~ s/:/=====double-colon=====/g;
          $remote_wwpn = $entry->{'Remote WWPN'};
        }
        if (defined $entry->{'Local WWPN'}) {
          $local_wwpn = $entry->{'Local WWPN'};
          $local_wwpn =~ s/:/=====double-colon=====/g;
        }
      }
    }
    # AIXErr:DateTime:Node Id:Resource Name:Resource Type:Description:LABEL:Sequence number:SENSE DATA:
    my $aix_err_output_line = ":aix_err:$datetime:$node_id:$resource_name:$resource_type:$description:$label:$seqnum:$remote_wwnn\/$remote_wwpn\/$local_wwpn";
    $all_output_err_lines .= $aix_err_output_line;
  }
  #print Dumper ($data);
  print("_DBG_: DATA:\n", Dumper( $data->{1123257} ), "\n") if ( $debug_aix_error >= 2 );
  return $all_output_err_lines;
}

sub parse_sense_fcperr {
  my $h = shift;
  my $line = shift;
  my $num = shift;
  #print( "__DBG__: $num - LINE: $line\n" );
  my @a = split( / /, $line );
  if( $num == 1 ) {
    #print( "__DBG__: $num - ARRAY: ", join('',@a) ,"\n" );
    $h->{'Local WWPN'} = join( "", $a[8], $a[9], $a[10], $a[11]);
  }
  if( $num == 7 ) {
    #print( "__DBG__: $num - ARRAY: ", join('',@a) ,"\n" );
    $h->{'Remote WWPN'} = join( "", $a[0], $a[1], $a[2], $a[3]);
    $h->{'Remote WWNN'} = join( "", $a[4], $a[5], $a[6], $a[7]);
  }
}

# FCA_ERRx - ADAPTER ERROR - IDENTIFIER:[D666A8C7]
sub parse_sense_fcaerr {
  my $h = shift;
  my $line = shift;
  my $num = shift;
  #print( "__DBG__: $num - LINE: $line\n" );
  my @a = split( / /, $line );
  if( $num == 5 ) {
    #print( "__DBG__: $num - ARRAY: ", join('',@a) ,"\n" );
    $h->{'Local WWPN'} = join( "", $a[11], $a[12], $a[13], $a[14]);
  }
  elsif( $num == 4 ) {
    #print( "__DBG__: $num - ARRAY: ", join('',@a) ,"\n" );
    $h->{'Remote WWPN'} = join( "", $a[8], $a[9], $a[10], $a[11]);
  }
  elsif( $num == 14 ) {
    $h->{'Remote WWNN'} = join( "", $a[11], $a[12], $a[13], $a[14]);
  }
}

sub read_stat {
  my $file = shift;
  my %st;
  my $fh;
  if ( -r $file ) {
    open( $fh, "<", $file ) or warn "aixerrpt: couldn't read specified stat file $file\n";
  }
  while( <$fh> ) {
    chomp;
    if( $_ =~ /^TIMESTAMP:(\d{10})$/ ) {
      $st{starttime} = $1;
    }
    elsif( $_ =~ /^SEQUENCE:(\d+)$/ ) {
      $st{sequence} = $1;
    }
  }
  close( $fh ) if ( defined $fh );
  #
  if ( defined $st{starttime} && $st{starttime} ) {
    return( \%st );
  }
  else {
    # Read start time from first entry in errpt and exit
    my $newst = new_stat();
    write_stat( $file, $newst );
  }
}

sub write_stat {
  my $file = shift;
  my $st = shift;
  my $fh;
  if ( -r $file ) {
    unlink( $file );
  }
  open( $fh, ">", $file ) or
    die "aixerrpt: couldn't write specified stat file $file\n";
  printf $fh ( "TIMESTAMP:%s\n", $st->{starttime} );
  if ( defined $st->{sequence} ) {
    printf $fh ( "SEQUENCE:%d\n", $st->{sequence} );
  }
  else {
    printf $fh ( "SEQUENCE:\n" );
  }
  close( $fh ) if ( defined $fh );
}

sub new_stat {
  my %st;
  $st{starttime} = undef;
  $st{sequence} = undef;
  # Read TIMESTAMP from first entry in errpt
  open (my $fh,'-|', "/usr/bin/errpt") or
    warn "aixerrpt: couldn't open errpt for reading!\n";
  while ( <$fh> ) {
    chomp;
    # IDENTIFIER TIMESTAMP  T C RESOURCE_NAME  DESCRIPTION
    if ( $_ =~ /^IDENTIFIER\s+TIMESTAMP/) { next; }
    # BFE4C025   0412162425 P H sysplanar0     UNDETERMINED ERROR
    elsif ( $_ =~ /^\w+\s+(\d{10})\s+/) {
      $st{starttime} = $1;
      last;
    }
  }
  close( $fh ) if ( defined $fh );
  #
  if ( not defined $st{starttime} ) {
    # set start time to current time
    # get the current time, in the form used by the errpt command ...
    $st{starttime} = sprintf( '%02d%02d%02d%02d%02d',
      sub { $_[4]+1, $_[3], $_[2], $_[1], $_[5] % 100 } -> (localtime) );
    return( \%st );
  }
  else {
    # Read Sequence Number from first entry in errpt
    open (my $fh,'-|', "/usr/bin/errpt -a -s " . $st{starttime} ) or
      warn "aixerrpt: couldn't open errpt for reading!\n";
    while ( <$fh> ) {
      chomp;
      # Sequence Number: 1993
      if ( $_ =~ /^Sequence Number:\s+(\d+)\s*/) {
        $st{sequence} = $1;
        last;
      }
    }
    close( $fh ) if ( defined $fh );
  }
  return( \%st );
}

sub ignore {
  my $identifier = shift;
  my @error;
  my @ignore;
  my $timestamp;
  my $ignore = 'no';
  foreach (@ignore) {
    # check to see if the entry is on the ignore list ...
    #  /usr/bin/errpt -s 0726093422 -j F31FFAC3 -N hdisk12
    #  /usr/bin/errpt -s $timestamp -j $identifie -N $resource
    if ($identifier =~ /$_/) { $ignore = 'yes' ; last }
  }
  unless  ($ignore eq 'yes') {
    # store the full entry in the array @error ...
    @error = `/usr/bin/errpt -a -s $timestamp`;
    # add a subject line to the @error array ...
    ## unshift (@error, "Subject: $hostname -- $resource -- $description\n");
  }
}

sub iostat {
  my $iostat_data_file = shift;
  my $IOSTAT           = shift;
  my @data_all;
  print "$IOSTAT 2>>$error_log\n";

  my $iostat_cmd = `$IOSTAT`;

  open( IOS, "> $iostat_data_file" ) || error( "Cannot open $iostat_data_file: $!" . __FILE__ . ":" . __LINE__ ) && exit 1;

  @data_all = split( "\n", $iostat_cmd );

  my $type           = "";
  my $adapter_name   = "";
  my $adapter_values = "";
  my @adapters;
  my @adapters_sol;
  my $header_line;
  my $iostat_cmd_s = "";
  if ( $os eq "SunOS" ) {
    ( undef, undef, $iostat_cmd_s ) = split( /extended device statistics/, $iostat_cmd );
    @data_all = split( "\n", $iostat_cmd_s );
  }
  foreach my $line (@data_all) {
    chomp $line;
    my $length = length($line);
    if ( $line =~ /^Adapter:/ )         { $type = "Adapter";        next; }
    if ( $line =~ /^Vadapter:/ )        { $type = "Vadapter";       next; }
    if ( $line =~ /^Disks:/ )           { $type = "Disk";           next; }
    if ( $line =~ /^Vtargets\/Disks:/ ) { $type = "Vtargets/Disks"; next; }
    if ( $os eq "SunOS" && $line !~ /^c/ ) {
      if ( grep /wsvc_t/, @data_all ) {    ###### other type of command output
        $type = "sol-adapter_type2";
      }
      else {
        $type = "sol-adapter_type1";
      }
      next;
    }

    #if ( $line =~ /^System configuration:|^System:|^--|^vsa|^vhost|^cd|^vfchost|^ / ) { next; }
    if ( $line =~ /^System configuration:|^System:|^--/ )                                                              { next; }
    if ( $line =~ /bps|Kbps/ && $line =~ /tps/ && $line =~ /bread|Kb_read|bkread/ && $line =~ /bwrtn|Kb_wrtn|bkwrtn/ ) { $header_line = $line; next; }
    if ( $length < 3 )                                                                                                 { next; }

    # sisi = SCSI like: sisioa0   Available 04-08 PCI-X Dual Channel U320 SCSI RAID Adapter
    if ( $type =~ /^Adapter|^Vadapter/ && $line =~ /^[a-z]/ ) {

      #
      # this seems to be unused, output $adapter_values is not used anywhere, utilisation is counted from hdisks anyway --PH
      #

      $line =~ s/\s+/,/g;
      my ( $name, undef ) = split( ",", $line );
      $adapter_name = $name;
      if ( $adapter_name =~ /^fcs|^vscsi|^sissas|^sisi/ ) {
        my ( $name, $bytes_sec, $io_sec, $bytes_sec_read, $bytes_sec_write ) = split( ",", $line );
        if ( $bytes_sec =~ /K$/ || $header_line =~ /Kbps/ ) { $bytes_sec =~ s/K$//g; $bytes_sec = $bytes_sec * 1000; }
        if ( $bytes_sec =~ /M$/ ) { $bytes_sec =~ s/M$//g; $bytes_sec = $bytes_sec * 1000000; }
        if ( $bytes_sec =~ /G$/ ) { $bytes_sec =~ s/G$//g; $bytes_sec = $bytes_sec * 1000000000; }
        if ( $bytes_sec =~ /T$/ ) { $bytes_sec =~ s/T$//g; $bytes_sec = $bytes_sec * 1000000000000; }
        if ( $bytes_sec_read =~ /K$/ || $header_line =~ /Kb_read/ ) { $bytes_sec_read =~ s/K$//g; $bytes_sec_read = $bytes_sec_read * 1000; }
        if ( $bytes_sec_read =~ /M$/ ) { $bytes_sec_read =~ s/M$//g; $bytes_sec_read = $bytes_sec_read * 1000000; }
        if ( $bytes_sec_read =~ /G$/ ) { $bytes_sec_read =~ s/G$//g; $bytes_sec_read = $bytes_sec_read * 1000000000; }
        if ( $bytes_sec_read =~ /T$/ ) { $bytes_sec_read =~ s/T$//g; $bytes_sec_read = $bytes_sec_read * 1000000000000; }
        if ( $bytes_sec_write =~ /K$/ || $header_line =~ /Kb_wrtn/ ) { $bytes_sec_write =~ s/K$//g; $bytes_sec_write = $bytes_sec_write * 1000; }
        if ( $bytes_sec_write =~ /M$/ ) { $bytes_sec_write =~ s/M$//g; $bytes_sec_write = $bytes_sec_write * 1000000; }
        if ( $bytes_sec_write =~ /G$/ ) { $bytes_sec_write =~ s/G$//g; $bytes_sec_write = $bytes_sec_write * 1000000000; }
        if ( $bytes_sec_write =~ /T$/ ) { $bytes_sec_write =~ s/T$//g; $bytes_sec_write = $bytes_sec_write * 1000000000000; }

        $adapter_values = "$name,$bytes_sec,$io_sec,$bytes_sec_read,$bytes_sec_write";
      }
    }

    if ( $line =~ /^hdisk/ && $adapter_name =~ /^fcs|^vscsi|^sissas|^sisi/ ) {

      # # iostat -Dsal 60 1
      #
      # ...
      #
      # Disks:                           xfers                                read                                write                                  queue
      # -------------------- -------------------------------- ------------------------------------ ------------------------------------ --------------------------------------
      # %tm    bps   tps  bread  bwrtn   rps    avg    min    max time fail   wps    avg    min    max time fail    avg    min    max   avg   avg  serv
      # act                                    serv   serv   serv outs              serv   serv   serv outs        time   time   time  wqsz  sqsz qfull
      # hdisk5                 4.0  12.3K   3.0   0.0   12.3K   0.0   0.0    0.0    0.0     0    0   3.0  11.6    0.0    0.0     0    0   0.0    0.0    0.0    0.0   0.0   0.0
      # hdisk3                 1.0  12.3K   3.0   0.0   12.3K   0.0   0.0    0.0    0.0     0    0   3.0   2.6    0.0    0.0     0    0   0.0    0.0    0.0    0.0   0.0   0.0
      # Adapter:                      xfers
      # -------------------- ---------------------------
      # bps    tps  bread  bwrtn
      # fcs1                 310.3K  69.0    0.0  310.3K
      #
      # ....
      #
      # fcs1 read latency = ((hdisk5:rps * hdisk5:avg-serv) + (hdisk3:rps * hdisk3:avg-serv)) / (hdisk5:rps + hdisk3:rps)
      # fcs1 write latency = ((hdisk5:wps * hdisk5:avg-serv) + (hdisk3:wps * hdisk3:avg-serv)) / (hdisk5:wps + hdisk3:wps)

      $line =~ s/\s+/,/g;
      my ( $name, undef, $bps, undef, $bread, $bwrtn, $rps, $avg_serv_read, undef, undef, undef, undef, $wps, $avg_serv_write, undef ) = split( ",", $line );
      if ( $adapter_name =~ /^vscsi/ ) {
        if ( $bps   =~ /K$/ ) { $bps   =~ s/K$//g; $bps   = $bps * 1000; }
        if ( $bps   =~ /M$/ ) { $bps   =~ s/M$//g; $bps   = $bps * 1000000; }
        if ( $bps   =~ /G$/ ) { $bps   =~ s/G$//g; $bps   = $bps * 1000000000; }
        if ( $bps   =~ /T$/ ) { $bps   =~ s/T$//g; $bps   = $bps * 1000000000000; }
        if ( $bread =~ /K$/ ) { $bread =~ s/K$//g; $bread = $bread * 1000; }
        if ( $bread =~ /M$/ ) { $bread =~ s/M$//g; $bread = $bread * 1000000; }
        if ( $bread =~ /G$/ ) { $bread =~ s/G$//g; $bread = $bread * 1000000000; }
        if ( $bread =~ /T$/ ) { $bread =~ s/T$//g; $bread = $bread * 1000000000000; }
        if ( $bwrtn =~ /K$/ ) { $bwrtn =~ s/K$//g; $bwrtn = $bwrtn * 1000; }
        if ( $bwrtn =~ /M$/ ) { $bwrtn =~ s/M$//g; $bwrtn = $bwrtn * 1000000; }
        if ( $bwrtn =~ /G$/ ) { $bwrtn =~ s/G$//g; $bwrtn = $bwrtn * 1000000000; }
        if ( $bwrtn =~ /T$/ ) { $bwrtn =~ s/T$//g; $bwrtn = $bwrtn * 1000000000000; }
        my $tps = $rps + $wps;
        $adapter_values = "$adapter_name,$bps,$tps,$bread,$bwrtn";
      }
      if ( $avg_serv_read  =~ /S$/ ) { $avg_serv_read  =~ s/S$//g; $avg_serv_read  = $avg_serv_read * 1000; }
      if ( $avg_serv_write =~ /S$/ ) { $avg_serv_write =~ s/S$//g; $avg_serv_write = $avg_serv_write * 1000; }
      if ( $type           =~ /Disk/ ) {
        my $read  = $rps * $avg_serv_read;
        my $write = $wps * $avg_serv_write;
        push( @adapters, "$adapter_values,$name,$read,$write,$rps,$wps\n" );
      }
    }

    if ( $line =~ /^c/ && $type =~ /sol-adapter_type1/ ) {
      $line =~ s/\s+/,/g;
      my ( $name, $iops_r, $iops_w, $recv_bytes, $trans_bytes, undef, undef, $resp_time ) = split( ",", $line );
      $recv_bytes  = $recv_bytes * 1000;
      $trans_bytes = $trans_bytes * 1000;
      my $zone_name1 = `zonename 2>/dev/null`;
      chomp $zone_name1;
      if ( $zone_name1 ne "global" ) { next; }

      #print "$adapter_values,$name,$iops_r,$iops_w,$recv_bytes,$trans_bytes\n";
      push( @adapters_sol, "$name,$iops_r,$iops_w,$recv_bytes,$trans_bytes,$resp_time\n" );
    }
    if ( $line =~ /^c/ && $type =~ /sol-adapter_type2/ ) {
      $line =~ s/\s+/,/g;
      my ( $name, $iops_r, $iops_w, $recv_bytes, $trans_bytes, undef, undef, $resp_time1, $resp_time2 ) = split( ",", $line );
      $recv_bytes  = $recv_bytes * 1000;
      $trans_bytes = $trans_bytes * 1000;
      my $resp_time  = $resp_time1 + $resp_time2;
      my $zone_name1 = `zonename 2>/dev/null`;
      chomp $zone_name1;
      if ( $zone_name1 ne "global" ) { next; }

      #print "$adapter_values,$name,$iops_r,$iops_w,$recv_bytes,$trans_bytes\n";
      push( @adapters_sol, "$name,$iops_r,$iops_w,$recv_bytes,$trans_bytes,$resp_time\n" );
    }
  }

  if (@adapters_sol) {
    foreach my $line (@adapters_sol) {
      chomp $line;
      my $char_l = "_ldom";
      my ( $name, $iops_r, $iops_w, $recv_bytes, $trans_bytes, $resp_time ) = split( ",", $line );
      print IOS"$name,$iops_r,$iops_w,$recv_bytes,$trans_bytes,$resp_time\n";
    }
  }

  @adapters = sort @adapters;
  my $last_found = "";

  foreach my $line (@adapters) {
    chomp $line;
    my ( $ada_name, $bytes_sec, $io_sec, $bytes_sec_read, $bytes_sec_write, undef, undef, undef, undef ) = split( ",", $line );

    my $found = "$ada_name";
    if ( $found eq $last_found ) {next}
    $last_found = $found;

    my @only_one_adapter_data = grep {/^$ada_name/} @adapters;
    my $resp_time_read        = 0;
    my $resp_time_write       = 0;
    my $sum_r_p_s             = 0;
    my $sum_w_p_s             = 0;

    my $sum_vscsi_bps   = 0;
    my $sum_vscsi_tps   = 0;
    my $sum_vscsi_bread = 0;
    my $sum_vscsi_bwrtn = 0;

    foreach my $line_l (@only_one_adapter_data) {
      chomp $line_l;
      my ( $a_name, $vscsi_bps, $vscsi_tps, $vscsi_bread, $vscsi_bwrtn, $disk_name, $read, $write, $rps, $wps ) = split( ",", $line_l );
      $resp_time_read  = $resp_time_read + $read;
      $resp_time_write = $resp_time_write + $write;
      $sum_r_p_s       = $sum_r_p_s + $rps;
      $sum_w_p_s       = $sum_w_p_s + $wps;

      $sum_vscsi_bps = $sum_vscsi_bps + $vscsi_bps;
      if ( $vscsi_bps > 0 ) {
        $sum_vscsi_tps = $sum_vscsi_tps + $vscsi_tps;
      }
      $sum_vscsi_bread = $sum_vscsi_bread + $vscsi_bread;
      $sum_vscsi_bwrtn = $sum_vscsi_bwrtn + $vscsi_bwrtn;
    }
    if ( $sum_r_p_s == 0 ) {
      $resp_time_read = 0;
    }
    else {
      $resp_time_read = $resp_time_read / $sum_r_p_s;
    }
    if ( $sum_w_p_s == 0 ) {
      $resp_time_write = 0;
    }
    else {
      $resp_time_write = $resp_time_write / $sum_w_p_s;
    }
    $resp_time_read  = sprintf( "%.1f", $resp_time_read );
    $resp_time_write = sprintf( "%.1f", $resp_time_write );
    if ( $ada_name =~ /^vscsi/ ) {
      print IOS "$ada_name,$sum_vscsi_bread,$sum_vscsi_bwrtn,$sum_r_p_s,$sum_w_p_s,$resp_time_read,$resp_time_write\n";
    }
    else {
      print IOS "$ada_name,$bytes_sec_read,$bytes_sec_write,$sum_r_p_s,$sum_w_p_s,$resp_time_read,$resp_time_write\n";
    }
  }
  close(IOS);
  return 1;
}

sub nfs_iostat {
  my $nfs_iostat_data_file = shift;
  my $NFS_IOSTAT           = shift;

  # nfsiostat error workround:
  # when is run as "60 1" the  just prinst on its start longrem stats, and finish after 60 secs
  # when running "60 2" then runs 2 minutes and get stats from last minute only --> it prolongs to run the agent about 1 minute for nothing
  # solution is "60 2" and sigalrm after 62 seconds, data must be passed with a temp file
  use File::Temp qw/ tempfile/;
  ( my $fh_tmp, my $tmp_name ) = tempfile( UNLINK => 1 );
  $| = 1;
  eval {
    local $SIG{ALRM} = sub { die "died in SIG ALRM: agent"; };
    my $nfs_timeout = $STEP + 2;
    print "$NFS_IOSTAT : timeout $nfs_timeout : $tmp_name \n";
    alarm($nfs_timeout);
    `$NFS_IOSTAT >$tmp_name 2>&1`;
    alarm(0);
  };
  $fh_tmp->flush;

  if ( ! -f $tmp_name ) {
    error("Cannot open nfsiostat temp file $tmp_name: $!" . __FILE__ . ":" . __LINE__);
    return 1;
  };

  my $nfs_iostat_cmd = "";

  if (open my $fh, "<", $tmp_name) {
    local $/;
    $nfs_iostat_cmd = <$fh>;
    close $fh;
  } else {
    print "Cannot open $tmp_name: $!\n";
    return 1;
  }

  if ($nfs_iostat_cmd =~ /No NFS/) {
    print "No NFS mount points were found\n";
    return 1;
  }
  print "$NFS_IOSTAT 2>>$error_log\n";

  open(my $fh, "> $nfs_iostat_data_file") || do {
    error("Cannot open $nfs_iostat_data_file: $!" . __FILE__ . ":" . __LINE__);
    return 1;
  };

  my %skip_first;     # track first mount block
  my $skip_block = 0; # check skip the block
  my $mount;
  my $read_done = 0;
  my @read_vals;

  my @lines = split("\n", $nfs_iostat_cmd);

  foreach my $line (@lines) {
    chomp($line);

    # detectiton new mount 
    if ($line =~ m#^(.*) mounted on (.*):$#) {
      $mount = $2;

      if (!exists $skip_first{$mount}) {
        # first block skip
        $skip_first{$mount} = 1;
        $skip_block = 1;
        $read_done = 0;
        @read_vals = ();
        next;
      }

      # second block 
      $skip_block = 0;
      $read_done = 0;
      @read_vals = ();
      next;
    }

    # skip block
    next if $skip_block;
    # skip header line
    next if $line =~ /^read:/ || $line =~ /^write:/ || $line =~ /^\s*$/;

    # first version of cmd - without queue/errors
    if ($line =~ /^\s*([\d.]+)\s+([\d.]+)\s+[\d.]+\s+(\d+)\s+\([\d.]+%\)\s+[\d.]+\s+([\d.]+)$/) {
      my ($ops, $kbs, $retrans, $exe) = ($1, $2, $3, $4);
      $exe = int($exe);
      if (!$read_done) {
        @read_vals = ($ops, $kbs, $retrans, $exe);
        $read_done = 1;
      } else {
        my $csv = join(",", $mount, @read_vals, $ops, $kbs, $retrans, $exe);
        print $fh "$csv\n";
        $read_done = 0;
        @read_vals = ();
      }
    }
    # second version of cmd - with queue/errors
    elsif ($line =~ /^\s*([\d.]+)\s+([\d.]+)\s+[\d.]+\s+(\d+)\s+\([\d.]+%\)\s+[\d.]+\s+([\d.]+)\s+([\d.]+)\s+(\d+)\s+\([\d.]+%\)$/) {
      my ($ops, $kbs, $retrans, $exe, $queue, $errors) = ($1, $2, $3, $4, $5, $6);
      $exe   = int($exe);
      $queue = int($queue);
      if (!$read_done) {
        @read_vals = ($ops, $kbs, $retrans, $exe, $queue, $errors);
        $read_done = 1;
      } else {
        my $csv = join(",", $mount, @read_vals, $ops, $kbs, $retrans, $exe, $queue, $errors);
        print $fh "$csv\n";
        $read_done = 0;
        @read_vals = ();
      }
    }
  }
  close($fh);
  return 1;
}

sub zfs_iostat {
  my ($zfs_iostat_data_file) = @_;

  my $zpool_bin = `which zpool 2>/dev/null`;
  chomp $zpool_bin;
  unless ($zpool_bin) {
    print "[ZFS_DEBUG] zpool command not found, exiting zfs_iostat\n";
    return 1;
  }

  my $cmd = "$zpool_bin status 2>&1";
  my $status_zpool = `$cmd`;
  chomp $status_zpool;

  # if zpool installed, but no pools available
  if ($status_zpool =~ /no\s+pools/i) {
    print "[ZFS_DEBUG] no pools available, exiting zfs_iostat\n";
    return 1;
  }

  # found all pools
  my @pool_lines = `$zpool_bin list -H 2>/dev/null`;
  chomp @pool_lines;
  my @pool_names = map { (split(/\s+/,$_))[0] } grep { $_ ne '' } @pool_lines;

  my @child_pids;
  my @tmp_files;

  # process group management
  setpgrp(0, 0);
  my $timeout = $STEP + 45;

  local $SIG{ALRM} = sub {
    kill '-15', $$;
    sleep 1;
    kill '-9', $$;
    exit(1);
  };

  eval {
    alarm($timeout);

    foreach my $pool (@pool_names) {
      (my $safe_pool = $pool) =~ s{[^A-Za-z0-9_.-]}{_}g;

      # filtering only the second block
      my $filter_second_block = sub {
        my ($pool_name, @lines) = @_;
        my @blocks;
        my @cur;

        foreach my $line (@lines) {
          if ($line =~ /^$pool_name\s+/ && @cur) {
            push @blocks, [@cur];
            @cur = ();
          }
          push @cur, $line;
        }
        push @blocks, [@cur] if @cur;
        return @blocks > 1 ? @{ $blocks[-1] } : @lines;
      };

      # pool list
      my $pid = fork();
      if (!defined $pid) {
        warn "[ZFS_DEBUG] fork failed for list $pool: $!";
      }
      elsif ($pid == 0) {
        setpgrp(0, 0);
        local $SIG{ALRM} = sub { die "ALRM\n" };
        alarm($STEP + 15);

        my $tmp = "/tmp/zfs_${safe_pool}_list.tmp";
        my @out = `$zpool_bin list -Hp $pool $STEP 2 2>/dev/null`;
        chomp @out;
        my @filtered = $filter_second_block->($pool, @out);

        open my $fh, ">", $tmp or do { print "[ZFS_DEBUG child] Cannot open $tmp: $!\n"; exit(1); };
        print $fh join("\n", @filtered);
        close $fh;
        exit(0);
      } else {
        push @child_pids, $pid;
        push @tmp_files, "/tmp/zfs_${safe_pool}_list.tmp";
      }

      # pool iostat
      $pid = fork();
      if (!defined $pid) {
        warn "[ZFS_DEBUG] fork failed for pool_iostat $pool: $!";
      }
      elsif ($pid == 0) {
        setpgrp(0, 0);
        local $SIG{ALRM} = sub { die "ALRM\n" };
        alarm($STEP + 15);

        my $tmp = "/tmp/zfs_${safe_pool}_pool_iostat.tmp";
        my @out = `$zpool_bin iostat -Hp $pool $STEP 2 2>/dev/null`;
        chomp @out;
        my @filtered = $filter_second_block->($pool, @out);

        open my $fh, ">", $tmp or do { print "[ZFS_DEBUG child] Cannot open $tmp: $!\n"; exit(1); };
        print $fh join("\n", @filtered);
        close $fh;
        exit(0);
      } else {
        push @child_pids, $pid;
        push @tmp_files, "/tmp/zfs_${safe_pool}_pool_iostat.tmp";
      }

      # vdev iostat -v
      $pid = fork();
      if (!defined $pid) {
        warn "[ZFS_DEBUG] fork failed for vdev_iostat $pool: $!";
      }
      elsif ($pid == 0) {
        setpgrp(0, 0);
        local $SIG{ALRM} = sub { die "ALRM\n" };
        alarm($STEP + 15);

        my $tmp = "/tmp/zfs_${safe_pool}_vdev_iostat.tmp";
        my @out = `$zpool_bin iostat -vHp $pool $STEP 2 2>/dev/null`;
        chomp @out;
        my @filtered = $filter_second_block->($pool, @out);

        open my $fh, ">", $tmp or do { print "[ZFS_DEBUG child] Cannot open $tmp: $!\n"; exit(1); };
        print $fh join("\n", @filtered);
        close $fh;
        exit(0);
      } else {
        push @child_pids, $pid;
        push @tmp_files, "/tmp/zfs_${safe_pool}_vdev_iostat.tmp";
      }
    }

    # arcstat global
    my $arcstat_bin = `which arcstat 2>/dev/null`;
    chomp $arcstat_bin;
    if ($arcstat_bin) {
      my $pid_arc = fork();
      if (!defined $pid_arc) {
        warn "[ZFS_DEBUG] fork failed for arcstat: $!";
      }
      elsif ($pid_arc == 0) {
        setpgrp(0, 0);
        local $SIG{ALRM} = sub { die "ALRM\n" };
        alarm($STEP + 15);
        my $tmp = "/tmp/zfs_arcstat.tmp";

        my $zpool_status = `$zpool_bin status 2>/dev/null`;
        my $arc_fields = ($zpool_status =~ /cache\s+/)
        ? 'time,hit%,ioh%,miss%,l2hit%,l2miss%,l2pref%'
        : 'time,hit%,ioh%,miss%';

        my @out = `$arcstat_bin -f $arc_fields $STEP 2 2>/dev/null`;
        chomp @out;

        my @blocks;
        my @cur;
        foreach my $line (@out) {
          if ($line =~ /^\d+:/ && @cur) {
            push @blocks, [@cur];
            @cur = ();
          }
          push @cur, $line;
        }
        push @blocks, [@cur] if @cur;
        my @filtered = @out;
        @filtered = @{ $blocks[-1] } if @blocks > 1;

        open my $fh, ">", $tmp or do { print "[ZFS_DEBUG child] Cannot open $tmp: $!\n"; exit(1); };
        print $fh join("\n", @filtered);
        close $fh;
        exit(0);
      } else {
        push @child_pids, $pid_arc;
        push @tmp_files, "/tmp/zfs_arcstat.tmp";
      }
    } else {
      print "zfs_arcstat not found, skipping\n";
    }

    # wait for children
    foreach my $pid (@child_pids) {
      my $wp = waitpid($pid, 0);
      if ($wp == -1) {
        warn "zfs_iostat waitpid failed for $pid: $!";
      }
    }

    alarm(0);
  };

  if ($@) {
    chomp($@);
    warn "[ZFS_DEBUG] Exception in zfs_iostat: $@\n";
    kill '-15', $$;
    sleep 1;
    kill '-9', $$;
  }
  # SIZE + HEALTH
  my %disk_info;

  my @list_v_lines = `$zpool_bin list -v 2>/dev/null`;
  chomp @list_v_lines;

  foreach my $line (@list_v_lines) {
    $line =~ s/^\s+|\s+$//g;  # trim leading/trailing spaces
    next if $line =~ /^NAME/;
    next if $line =~ /^$/;

    my @cols = split /\s+/, $line;

    my $name   = defined $cols[0] ? $cols[0] : '';
    my $size   = defined $cols[1] ? $cols[1] : '';
    my $health = defined $cols[-1] ? $cols[-1] : '';  # HEALTH is last column

    # skip pools, vdevs, cache, spare
    next if $name =~ /^(pool\d*|mirror\S*|raidz\S*)$/;

    $disk_info{$name} = [$size, $health];
  }


  my $message = '';

  # build mapping (pool → vdev → disk)
  my @status = `$zpool_bin status -v 2>/dev/null`;
  my (%map, $current_pool, $current_vdev, $section_type);

  foreach my $line (@status) {
    chomp $line;
    $line =~ s/^\s+//;  # trim left spaces
    next if $line =~ /^(state|config|NAME|errors|scan|replacing|logs|replacing)$/;
    next if $line =~ /^$/;

    # detect pool name
    if ($line =~ /^pool:\s*(\S+)/) {
      $current_pool = $1;
      next;
    }

    my ($name) = ($line =~ /^(\S+)/);
    next unless $name;

    # catch cache/spare section markers
    if ($name =~ /^(cache|spare|spares)$/i) {
      $section_type = lc($1);
      $current_vdev = $section_type;
      next;
    }

    # detect top-level vdevs (mirror, raidz, etc.)
    if ($name =~ /^(mirror|raidz\d*-\d+)/) {
      $current_vdev = $name;
      $section_type = undef;
      next;
    }

    # detect disk entries (sdb, sdc, etc.)
    if ($name =~ /^sd[a-z]+\d*$/i || $name =~ /^nvme\d+n\d+$/i) {
      if ($section_type) {
        # cache or spare disk
        push @{ $map{$current_pool}{$section_type} }, $name;
      } else {
        # regular vdev disk
        push @{ $map{$current_pool}{$current_vdev} }, $name;
      }
      next;
    }
  }

  # Build output message
  foreach my $pool (sort keys %map) {
    my @vdevmap;
    foreach my $v (sort keys %{ $map{$pool} }) {
      push @vdevmap, "$v(" . join(',', @{ $map{$pool}{$v} }) . ")";
    }
    $message .= ":zfs_map:$pool:" . join(',', @vdevmap) . "::::\::";
  }

  foreach my $tmp (@tmp_files) {
    next unless -f $tmp;
    open my $fh, "<", $tmp or next;
    while (my $line = <$fh>) {
      chomp $line;
      next if $line =~ /^\s*$/;
      if ($tmp =~ /_list\.tmp$/) {
        my ($pool_name, $size, $alloc, $free, $ckpt, $expand, $frag, $cap, $dedup, $health, $altroot) = split(/\s+/, $line);
        $message .= ":zfs_pool:$pool_name:$size:$alloc:$free:$cap:$frag:$dedup:$health";
      }
      elsif ($tmp =~ /_pool_iostat\.tmp$/) {
        my ($dev_name, $alloc, $free, $r_iops, $w_iops, $r_bps, $w_bps) = split(/\s+/, $line);
        $message .= ":zfs_pool_iostat:$dev_name:$r_iops:$w_iops:$r_bps:$w_bps:\::";
      }
      elsif ($tmp =~ /_vdev_iostat\.tmp$/) {
        if ($line =~ /^\s*(\S+)\s+/) {
          my ($dev_name) = ($1);
          my ($dev, @vals) = split(/\s+/, $line);
          next if grep { $_ eq $dev_name } @pool_names; # only volumes/disks
          my ($alloc,$free,$r_ops,$w_ops,$r_bps,$w_bps) = ( $vals[0] // '', $vals[1] // '', $vals[2] // 0, $vals[3] // 0, $vals[4] // 0, $vals[5] // 0 );
          if ($dev_name =~ /^(mirror|raidz)/) {
            $message .= ":zfs_volume:$dev_name:$alloc:$free:$r_ops:$w_ops:$r_bps:$w_bps:";
          } else {
            my ($size, $health) = ('', '');
            if (exists $disk_info{$dev_name}) {
              ($size, $health) = @{ $disk_info{$dev_name} };
            }
            $size     =~ s/[A-Za-z]+$// if defined $size;
            $size   = defined $size   ? $size   : '';
            $health = defined $health ? $health : '';
            $message .= ":zfs_disk:$dev_name:$r_ops:$w_ops:$r_bps:$w_bps:$size:$health:";
          }
        }
      }
      elsif ($tmp =~ /arcstat\.tmp$/) {
        my @fields = split(/\s+/, $line);
        shift @fields if $fields[0] && $fields[0] =~ /^\d/;
        while (@fields < 6) { push @fields, '' }
        @fields = @fields[0..5];
        $message .= ":zfs_arc:" . join(':', @fields) . ":\:";
      } else {
        $message .= ":zfs_raw:$line";
      }
    }
    close $fh;
    #print "====$tmp=====\n";
    unlink $tmp;
  }

  # Missing physical disks only (skip cache/spare)
  foreach my $disk (sort keys %disk_info) {
    # if exist disk next
    next if $message =~ /:zfs_disk:$disk:/;
    # if cache/spare next, only disks
    next if $disk =~ /^(cache|spare)$/i;

    my ($size, $health) = @{ $disk_info{$disk} };
    $size =~ s/[A-Za-z]+$// if defined $size;
    $size   = defined $size   ? $size   : '';
    $health = defined $health ? $health : '';

    $message .= ":zfs_disk:$disk:0:0:0:0:$size:$health:";
  }

  open my $out, ">", $zfs_iostat_data_file or do { warn "[ZFS_DEBUG] Cannot open $zfs_iostat_data_file: $!"; return 1; };
  print $out $message;
  close $out;

  return 1;
}

sub send_data    # send data to all accepting servers from cmd line (ARGV)
{
  my $store_file       = shift;
  my $protocol_version = shift;

  if ( !-f "$store_file" ) {
    return 0;    # nothing to re-send
  }

  # 2nd randomization to spread again a bit connections from the OS agents in time 1 - 49 seconds
  my $random = substr( int( rand(50) ), 0, 2 );    # substr just to be sure
  if ( isdigit($random) && $random > 0 ) {
    print "Agent send slp: sending wait: $random\n";
    sleep($random);
  }

  my $new_store_file    = $store_file;
  my $orig_file_to_send = "$store_file" . "orig";
  copy( "$store_file", "$orig_file_to_send" ) || error( "Cant cp $store_file $orig_file_to_send: $!" . __FILE__ . ":" . __LINE__ ) && return 1;

  my $numArgs = $#ARGV + 1;

  # print "you gave $numArgs command-line arguments:\n";
  my $ret = 0;

  foreach my $argnum ( 0 .. $#ARGV ) {
    print "OS/HMC agent working for server: $ARGV[$argnum]\n";
    ( $host, $port ) = split( ":", $ARGV[$argnum] );
    if ( !defined $port ) {
      $port = $port_standard;
    }
    $host =~ s/ .*$//g;
    $port =~ s/ .*$//g;

    # this make no sense --PH
    #if ( ( !isdigit($port) ) || ( $port < 1000 ) ) {
    #  error( "error setting port $port " . __FILE__ . ":" . __LINE__ );
    #  next;
    #}

    #print "host:$host - port:$port -\n";

    if ( $argnum > 0 ) {

      # prepare data and log-, err-, timestamp- files for next accepting server
      my $agent = "agent-hmc";
      if ( index( $error_log, $agent ) == -1 ) {    # is it hmc?
        $agent = "agent";
      }
      if ($hvmsh_host) {
        $agent = "agent-hitachi-$hvmsh_host";
      }

      $error_log           = "$base_dir/lpar2rrd-$agent-$host-$user_name.err";
      $timestamp_file      = "$base_dir/lpar2rrd-$agent-$host-$user_name.stamp";
      $timestamp_file_send = "$base_dir/lpar2rrd-$agent-$host-$user_name.stamp-send";
      print "$error_log\n $timestamp_file\n $timestamp_file_send\n";

      my $ret = `touch $timestamp_file`;            # to keep same routine for all timestamp files
      if ($ret) {
        error( "$ret " . __FILE__ . ":" . __LINE__ );
      }

      $new_store_file = "$base_dir/lpar2rrd-$agent-$host-$user_name.txt";

      # append orig_lines to new_store_file or create new file if there is not
      # open( FR,  "< $orig_file_to_send" ) || error( "Can't open $orig_file_to_send: $! " . __FILE__ . ":" . __LINE__ ) && return 1;
	  open( FR,  "< $lastly_created_lines" ) || error( "Can't open $store_file: $! " . __FILE__ . ":" . __LINE__ ) && return 1;
      open( FRA, ">> $new_store_file" )   || error( "Can't open $new_store_file: $! " . __FILE__ . ":" . __LINE__ )    && next;
      while ( my $line = <FR> ) {
        print FRA $line;
        print "for $ARGV[$argnum] appended line ,$line,\n";
      }
      close(FRA);
      close(FR);
      # trim files if necessary
      my $trimlogs_run = "$base_dir/lpar2rrd-$agent-$host-$user_name.stamp-trimlogs";
      my $trim_file_time_diff = file_time_diff("$trimlogs_run");
      if ( !-f "$trimlogs_run" || $trim_file_time_diff > 86400 ) {    # once per day
		print "---- trim $new_store_file\n";
        trimlog( "$new_store_file", "$trim_max_lines_store" );        # limit = 43000 lines
        my $ret = `touch $trimlogs_run`;    # to keep same routine for all timestamp files
        if ($ret) {
          error( "$ret " . __FILE__ . ":" . __LINE__ );
        }
      }
    }
    print "store file for sending is $new_store_file\n";
    my $ret = send_data_for_one_server( $new_store_file, $protocol_version );
  }
  unlink $lastly_created_lines;
  return $ret;
}

sub send_data_for_one_server {
  my $store_file       = shift;
  my $protocol_version = shift;
  my $client;

  if ( !-f "$store_file" ) {
    error( "Error: $store_file does not exist - nothing to re-send: $! " . __FILE__ . ":" . __LINE__ );
    return 0;    # nothing to re-send
  }

  open( FR, "< $store_file" ) || error( "Can't open $store_file: $! " . __FILE__ . ":" . __LINE__ ) && return 1;

  my $server_saved_time = keep_time( 0, "read" );

  if ( connect_server( \$client, $protocol_version ) == 1 ) {
    return 1;
  }

  $last_ok_time = 0;
  my $send_ok         = 1;
  my $counter         = 0;
  my $counter_err     = 0;
  my $store_file_indx = 0;
  my $line_refused    = "";

  open( FW, "> $store_file-tmp" ) || error( "Can't open $store_file: $! " . __FILE__ . ":" . __LINE__ ) && return 1;

  while ( my $line = <FR> ) {

    # read line by line, do not place whole file into the memory as big files fails with ulimit issue
    # use while as foreach internally needs the whole file
    #print "001 $line";
    chomp($line);
    if ( $line eq '' ) {
      next;
    }
    my $time_send_max = 2000000000;
    if ( index( $nmon_data, "ext_nmon_" ) != -1 ) {    # external NMON
      $time_send_max = 5000000000                      # for ext NMON could be
    }
    ( my $server_send, my $tr2, my $tr3, my $time_send ) = split( /:/, $line );
    if ( isdigit($time_send) == 0 || $time_send < 1000000000 || $time_send > $time_send_max ) {

      # time is wrong --> the line is corrupted, skip it
      if ( $counter_err == 0 ) {
        error("Skipping line due to wrong time ($time_send) : $line");
      }
      $counter_err++;
      next;
    }

    if ( $time_send <= $server_saved_time && $server_saved_time > 0 ) {

      # Send only new data from last saved one
      if ( $counter_err == 0 ) {
        error("Skipping line, it has been already sent ($time_send < $server_saved_time) : $line");
      }
      $counter_err++;
      next;
    }

    if ( $server_send eq '' ) {

      # server is wrong --> the line is corrupted, skip it
      if ( $counter_err == 0 ) {
        error("Skipping line due to wrong server ($server_send) : $line");
      }
      $counter_err++;
      next;
    }

    if ( $send_ok == 1 ) {
      $counter++;

      # print   "at this moment sending $line\n";  # print the line
      if ($using_ssl) {
        print $client "$line\n";
      }
      else {
        $client->send("$line\n");
      }
      if ( receive_data( \$client, $line ) == 0 ) {    # receive an answer
        $send_ok         = 0;                    # problem during sending --> keep do not sent rows
        $store_file_indx = 1;
        $line_refused    = $line;

        # keep here not send rows
        print FW "$line\n";

        #print "003\n";
      }
      else {
        $last_ok_time = $time_send;
      }
    }
    else {
      print FW "$line\n";
    }

    if ( $counter > 1000 ) {

      # save last OK time every 1000 rows
      $counter = 0;
      keep_time( $last_ok_time, "write" );
    }
  }
  close(FR);

  # save last OK record time
  keep_time( $last_ok_time, "write" );

  $client->close();

  unlink("$store_file");

  if ( $counter_err > 0 ) {
    error("Number of errors in the input file: $counter_err");
  }

  close(FW);
  if ( -f "$store_file-tmp" && $store_file_indx > 0 ) {

    #print "004\n";
    rename( "$store_file-tmp", "$store_file" );
    error( "Error: Not all data has been sent out, refused line: $line_refused " . __FILE__ . ":" . __LINE__ );
    return 1;    # do not continue in sending actual one as there is a problem
  }

  return 0;
}

sub store_for_later_transfer {
  my $message                = shift;
  my $store_not_confirm_file = shift;

  print "$store_not_confirm_file\n $message\n";
  open( FS, ">> $store_not_confirm_file" ) || error( "Can't open $store_not_confirm_file: $! " . __FILE__ . ":" . __LINE__ ) && return 1;
  print FS "$message\n";
  close(FS);
  return 1;
}

#
# run every $RUN_TIME and in case random() return 0 to spread agent load from many lpars across time
#
sub run_now {
  my $act_time_u     = shift;
  my $timestamp_file = shift;
  my $run_time       = shift;

  if ( $run_time == 1 ) {
    my $ret = `touch $timestamp_file`;    # first run after the upgrade
    if ($ret) {
      error( "$ret" . __FILE__ . ":" . __LINE__ );
    }
    print "Agent send     : yes : forced by -d \n";

    # debug run with option -w means "now"
    return 1;
  }

  if ( !-f $timestamp_file ) {
    my $ret = `touch $timestamp_file`;    # first run after the upgrade
    print "Agent send     : no : initial\n";
    if ($ret) {
      error( "$ret" . __FILE__ . ":" . __LINE__ );
    }
    return 0;
  }
  else {
    my $last_send_time = ( stat("$timestamp_file") )[9];
    my $random         = substr( int( rand(10) ), 0, 1 );    # substr just to be sure
    if ( isdigit($random) == 0 ) {
      $random = 1;
    }
    my $next_time = $last_send_time + $run_time;
    if ( $next_time > $act_time_u || $random != 1 ) {
      if ( ( $act_time_u - $last_send_time ) > $MAX_RANDOM_TIME ) {

        # send it out after 10 minutes the latest 600 secs
        print "Agent send 10m : sending data now (act_time=$act_time_u, last_send_time=$last_send_time, next_time=$next_time, random=$random)\n";
        my $ret = `touch $timestamp_file`;
        if ($ret) {
          error( "$ret" . __FILE__ . ":" . __LINE__ );
        }
        return 1;
      }
      else {
        print "Agent send     : not sending data this time (act_time=$act_time_u, last_send_time=$last_send_time, next_time=$next_time, random=$random)\n";
        return 0;
      }
    }
    else {
      print "Agent send     : sending data now (act_time=$act_time_u, last_send_time=$last_send_time, next_time=$next_time, random=$random)\n";
      my $ret = `touch $timestamp_file`;
      if ($ret) {
        error( "$ret" . __FILE__ . ":" . __LINE__ );
      }
      return 1;
    }
  }
}

# error handling
sub error {
  my $text     = shift;
  my $act_time = localtime();

  if ( $text =~ m/remote host did not respond within the timeout period/ ) {
    $text .= " : firewall issue??";
  }
  if ( $text =~ m/ A remote host refused an attempted connect/ ) {
    $text .= " : LPAR2RRD daemon does not seem to be running??";
  }

  #print "ERROR          : $text : $!\n";
  print STDERR "$act_time: $text\n";

  open( FE, ">> $error_log" ) || return 1;    # ignore error here
  print FE "$act_time: $text\n";

  #chmod 0777, "$error_log";

  return 1;
}

sub isdigit {
  my $digit = shift;
  my $text  = shift;

  if ( $digit eq '' ) {
    return 0;
  }
  if ( $digit eq 'U' ) {
    return 1;
  }

  my $digit_work = $digit;
  $digit_work =~ s/[0-9]//g;
  $digit_work =~ s/\.//;
  $digit_work =~ s/^-//;
  $digit_work =~ s/e//;
  $digit_work =~ s/\+//;
  $digit_work =~ s/\-//;

  if ( length($digit_work) == 0 ) {

    # is a number
    return 1;
  }

  #if (($digit * 1) eq $digit){
  #  # is a number
  #  return 1;
  #}

  # NOT a number
  return 0;
}

# It saves or reads the last saved record from the server side
sub keep_time {
  if ( length $nmon_data > 0 ) { return 0 }    # nothing for NMON

  my $saved_time = shift;
  my $read_write = shift;

  if ( $read_write =~ m/read/ ) {
    if ( -f $timestamp_file_send ) {
      open( FS, "< $timestamp_file_send" ) || error( "Can't open $timestamp_file_send: $! " . __FILE__ . ":" . __LINE__ ) && return 0;
      my @lines = <FS>;
      close(FS);
      my $stime = 0;
      foreach my $line (@lines) {

        #print "001 $line";
        $stime = $line;
        chomp($stime);
        last;
      }
      if ( !($stime eq '') && isdigit($stime) ) {
        return $stime;
      }
      else {
        return 0;    # something wrong
      }
    }
    else {
      return 0;      # not yet saved time
    }
  }
  else {

    if ( $saved_time == 0 ) {
      return 0;      # skip it when it is 0
    }
    open( FS, "> $timestamp_file_send" ) || error( "Can't open $timestamp_file_send: $! " . __FILE__ . ":" . __LINE__ ) && return 0;
    print FS "$saved_time";
    close(FS);
  }

  return 1;
}

#
#---- NMON analyse passage
#

my $signal_file;
my $agent_nmon_file;    #same as $store_not_confirm_file
my $semaphore_file;
my @files;
my @files_tmp;
my $interval;
my $server;
my $lpar;
my $lpar_id;
my $paging_space;
my $paging_percent;
my $message;
my $proc_clock;
my @ent;
my @ent_inx;
my @fcs;
my @fcs_inx;
my $entitled_proc;
my $zzzz_this;
my $last_zzzz_this;
my @part1;
my $os_type;
my $print_line;
my $nmon_external_time;
my @fcf;
my @fcf_inx;
my @sea_info;
my @sea_inx;
my @sea;
my @phys_log_eth_sea;

# $nmon_data is a directory, where user can copy/has
#   historical and also today`s up to date nmon files
#
# -- algorithm
#
#  read files from directory $nmon_data
#  empty dir ? -> exit
#
#  make list of files
#  omit these : *.gz, *.Z, *.bz2 and not nmon files
#
#  sort according 'server-lpar-firsttime-file_name'
#  there is a signal file
#  containing lines 'server-lpar-lastsenttime-file_name'
#  if not -> create signal file
#
# LOOP:
#    take ( next - 1st ) file from files = 'orig'
#    is this SRV & LPR in signal file?
#    NO send it (1st time for this SRV&LPR)
#    |  YES: is orig time < signal time
#    |       NO  send it (not sent yet, probably new day)
#    |       | YES: look ahead next file:
#    |       |     same SRV & LPR ?
#    |       |     NO send it (maybe today up to date file)
#    |       |     |   YES: look ahead orig time < signal time --- Yes LOOP old files to skip
#    |       |     |   NO send it
#    send it & save 'server-lpar-lastsenttime-file_name'
#    prepare to send new items from orig starting from signal time
#              ( it also could be nothing new ! )
#  End LOOP
#
#  calls sub to send after each file during the run
#

sub send_nmon_data {

  $print_line         = "\n";
  $nmon_external_time = "";
  if ( index( $nmon_data, "ext_nmon_" ) != -1 ) {    # external NMON
    $print_line = "<BR>";

    # get nmon_external_time from e.g. '/tmp/ext_nmon_1402555094/external_nmon_file_name'
    ( undef, $nmon_external_time ) = split( "ext_nmon_", $nmon_data );
    ( $nmon_external_time, undef ) = split( "\/", $nmon_external_time );
    if ( ( !isdigit($nmon_external_time) ) || ( $nmon_external_time < 1400000000 ) ) {
      error( "error \$nmon_external_time $nmon_external_time " . __FILE__ . ":" . __LINE__ );
      $nmon_external_time = "1444555666";
    }
  }
  my $numArgs = $#ARGV + 1;

  # print "thanks, you gave me $numArgs command-line arguments:$print_line";
  my $ret = 0;

  foreach my $argnum ( 0 .. $#ARGV ) {

    print "NMON working for server: $ARGV[$argnum] $print_line";
    ( $host, $port ) = split( ":", $ARGV[$argnum] );
    if ( !defined $port ) {
      $port = $port_standard;
    }
    if ( ( !isdigit($port) ) || ( $port < 1000 ) ) {
      error( "error setting port $port " . __FILE__ . ":" . __LINE__ );
      $ret = 1;
      next;
    }

    # print "\$port is $port $print_line";
    $signal_file     = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name-time_file.txt";
    $agent_nmon_file = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.txt";
    $error_log       = "$base_dir/lpar2rrd-agent-nmon-$host-$user_name.err";

    $ret = send_nmon_data_for_one_server();
  }
  return $ret;
}

sub send_nmon_data_for_one_server {

  #my  $DEBUG=3;
  #    $DEBUG=0;

  if ( !-f $signal_file ) {
    print "created file $signal_file " . __FILE__ . ":" . __LINE__ . "\n";
    my $output = `touch "$signal_file" 2>&1`;
    if ($output) {
      error( "$output " . __FILE__ . ":" . __LINE__ );
      return 1;
    }
  }

  opendir( DIR, $nmon_data ) || error( "can't opendir $nmon_data: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  @files_tmp = grep { !/^\./ && !/\.gz|\.Z|\.bz2/ && -f "$nmon_data/$_" } readdir(DIR);
  closedir DIR;

  if ( !defined $files_tmp[0] ) { return 1 }

  print "files NMON read:\n", join( "\n", @files_tmp ), "\n";

  fill_files();    # create sorted list of NMON files

  print "NMON read sorted files:\n", join( "\n", @files ), "\n";

  if ( !defined $files[0] ) {    # nothing to send
    return 0;
  }
  open( DF_IN, "< $signal_file" ) || error( "Cant open for reading $signal_file: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  my @signal_lines = <DF_IN>;
  close DF_IN;

  # LOOP:
  for my $i ( 0 .. $#files ) {
    ( my $data_origin_1, my $timefile ) = split( /_TIME_/, $files[$i] );
    my ( $orig_time, $orig_file ) = split( /_XOR_/, $timefile );
    my $sent_time = 0;

    my @matches = grep {/\Q$data_origin_1\E/} @signal_lines;
    if ( @matches > 1 ) {
      error("Two items $data_origin_1 in \@signal_lines not possible");
    }
    my ($index) = grep { $signal_lines[$_] =~ /\Q$data_origin_1\E/ } 0 .. $#signal_lines;
    if ( !defined $index ) { $index = -1 }
    print "index in signal array for $data_origin_1 is $index\n";
    if ( $index != (-1) ) {    # exists
      ( undef, $sent_time ) = split( /_TIME_/, $signal_lines[$index] );
      ( $sent_time, undef ) = split( /_XOR_/, $sent_time );
      if ( $orig_time < $sent_time ) {    # if NO send it
                                             # look ahead file
        if ( defined $files[ $i + 1 ] ) {    # if NO send it
          ( my $data_origin_2, my $timefile ) = split( /_TIME_/, $files[ $i + 1 ] );
          if ( $data_origin_1 eq $data_origin_2 ) {    # if NO send it
            ( my $look_ahead_orig_time, undef ) = split( /_XOR_/, $timefile );
            if ( $look_ahead_orig_time < $sent_time ) {
              next

                # skip old files
            }
          }
        }
      }
    }
    print "here all cases for send for $orig_file\n";

    my $result = prepare_nmon_file( "$nmon_data/$orig_file", $sent_time );

    # 0 - err, $nmon_unix_sent_time of last data

    print "final prepared data time $result\n";
    if ( $result == 0 ) {
      print "err preparing file to send $orig_file \n";
      return 1;
    }

    if ( call_agent_send($agent_nmon_file) == 1 ) {
      return 1;
    }

    # write signal to signal array
    my $new_signal = "$data_origin_1" . "_TIME_$result" . "_XOR_$orig_file";
    if ( $index == (-1) ) {
      push @signal_lines, $new_signal;
    }
    else {
      $signal_lines[$index] = $new_signal;
    }

    # save signal array to signal file after each successful sent file
    open( DF_OUT, "> $signal_file" ) || error( "Cant open for writing $signal_file: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
    foreach (@signal_lines) {
      chomp($_);
      print DF_OUT "$_\n";    # save each entry from signal array to the file
    }
    close DF_OUT;

  }    # end of cycle for

  return 0;
}

sub call_agent_send {
  my $file_to_send = shift;

  #my  $DEBUG=3;
  #    $DEBUG=0;

  if ( !-f $file_to_send ) {
    error("call_agent_send: file to send does not exist");
    return 1;
  }
  my $file_len = -s $file_to_send;
  print "call_agent_send: file to send has $file_len length\n\n";
  return 0 if $file_len == 0;

  my $ret = send_data_for_one_server( $file_to_send, $protocol_version );

  return $ret;
}

sub fill_files {

  # @files_tmp  is a list of files
  # if file is NMON file
  #  -> create name = "SRV_server_LPR_lpar_TIME_time_XOR_file-name"
  #  -> fill @files with these names
  #  sort @files

  #my  $DEBUG=3;
  #    $DEBUG=0;

  my @new_files;

  for my $file (@files_tmp) {
    my $server                 = "";
    my $lpar                   = "";
    my $unix_time_first_record = 0;

    open( DF_IN, "< $nmon_data/$file" ) || error( "Cant open for reading $file: $!" . __FILE__ . ":" . __LINE__ ) && return 0;
    ( $server, $lpar, $unix_time_first_record ) = read_first_part( 'yestime', "$nmon_data/$file" );
    print "$server,$lpar,$unix_time_first_record are from file $file\n";
    close DF_IN;

    if ( $lpar ne "" && $server ne "" && $unix_time_first_record != 0 ) {
      my $xstr = "SRV_$server" . "_LPR_$lpar" . "_TIME_$unix_time_first_record" . "_XOR_$file";
      print "push new name $xstr\n";
      push @new_files, $xstr;
    }
    else {
      print "not NMON file $file\n";    #if $DEBUG == 3;
    }
  }
  @files = sort @new_files;
}

sub prepare_nmon_file {
  my $file_to_proces  = shift;
  my $unix_time_start = shift;          # preparing from

  $message = "";
  my $nmon_unix_sent_time = 0;

  #my    $DEBUG=3;
  # my    $DEBUG=0;

  # prepares file to send to $agent_nmon_file
  # $agent_nmon_file    # contains filename for data sending to daemon
  # returns
  # 0 - err, nmon_unix_sent_time of last data

  print "preparing data from $file_to_proces starting after time $unix_time_start\n";    # if $DEBUG == 3;

  open( FFA, ">> $agent_nmon_file" ) || error( "Cant open for writing $agent_nmon_file: $!" . __FILE__ . ":" . __LINE__ ) && return 0;

  open( DF_IN, "< $file_to_proces" ) || error( "Cant open for reading $file_to_proces: $!" . __FILE__ . ":" . __LINE__ ) && return 0;
  ( $server, $lpar, $lpar_id, $paging_space, $paging_percent, $proc_clock ) = read_first_part( 'notime', $file_to_proces );

  #  print "$server,$lpar,$lpar_id,$paging_space,$paging_percent,$proc_clock,$unix_time_start,\n" if $DEBUG == 3;

  if ( $unix_time_start == 0 ) { $unix_time_start++ }
  ;                                                                                      # just for starting cycle
  while ($unix_time_start) {
    $unix_time_start = read_one_run( $unix_time_start, $file_to_proces );
    if ( $unix_time_start > 0 ) {
      $nmon_unix_sent_time = $unix_time_start;
    }
  }
  close FFA;
  close DF_IN;
  return $nmon_unix_sent_time;
}

sub read_first_part {
  my $yestime        = shift;    # if 'yestime' then only for test
  my $file_to_proces = shift;
  #  my $DEBUG = 0;

  $interval         = 60;
  $server           = "";
  $lpar             = "";
  $lpar_id          = "0";
  $paging_space     = "false";
  $paging_percent   = "";
  $proc_clock       = "";
  @ent              = ();
  @ent_inx          = ();
  @fcs              = ();
  @fcs_inx          = ();
  @part1            = ();
  @fcf              = ();
  @fcf_inx          = ();
  @sea              = ();
  @sea_inx          = ();
  @phys_log_eth_sea = ();

  my $lpar_n       = "";
  my $lpar_runname = "";    #for AIX 5.3.

  # AAA,SerialNumber,65383FP
  # AAA,LPARNumberName,5,BSRV21LPAR5-pavel
  # AAA,MachineType,IBM,8233-E8B

  # BBBP,001,lsconf,"System Model: IBM,8233-E8B"
  # BBBP,002,lsconf,"Machine Serial Number: 65383FP"

  # BBBP,1092,lsconf,"Modčle de systčme : IBM,9119-595"
  # BBBP,1093,lsconf,"Numéro de sé/LPARrie de la machine : 8323EAD"

  # BBBP,007,lsconf,"Processor Clock Speed: 3000 MHz"
  # BBBP,1098,lsconf,"Fréquence d'horloge du processeur : 2102 MHz"

  # BBBP,010,lsconf,"LPAR Info: 5 BSRV21LPAR5-pavel"
  # BBBP,1101,lsconf,"Infos LPAR : 27 P127-751"

  # BBBP,027,lsconf,"Paging Space Information"
  # BBBP,028,lsconf,"       Total Paging Space: 4096MB"
  # BBBP,029,lsconf,"       Percent Used: 12%"

  # BBBP,1119,lsconf,"	Espace de pagination total : 6112MB"
  # BBBP,1120,lsconf,"	Pourcentage utilisé : 1%"

  # BBBN,001,en2,1500,2047,Standard Ethernet Network Interface
  # BBBN,002,en4,1500,1024,Standard Ethernet Network Interface

  # BBBP,536,lsattr -El sys0,"ent_capacity    1.00    Entitled processor capacity   False"

  my $line;

  while ( $line = <DF_IN> ) {
    $line =~ s/\r[\n]*/\n/gm;    # all platforms Unix, Windows, Mac
    chomp($line);
    push @part1, $line;
    if ( @part1 > 2000000 ) {
      error("err reading first part of NMON file (more than 2000000 lines) $file_to_proces ");
      return ( "", "", 0 );
    }
    last if $line =~ "ZZZZ,T";
  }
  if ( $line !~ "ZZZZ,T" ) {
    error("unable reading first part of NMON file (didnt find ZZZZ,T) $file_to_proces ");
    return ( "", "", 0 );
  }
  $zzzz_this      = $line;
  $last_zzzz_this = $line;

  # all systems checks
  my $lines_read = @part1;
  print "part1 lines read $lines_read until $last_zzzz_this\n";

  #  CPU_ALL,CPU Total bsrpdev0035,User%,Sys%,Wait%,Idle%,Busy%,CPUs#
  my @matches     = grep {/^CPU_ALL/} @part1;
  my $match_count = scalar @matches;
  if ( $match_count != 1 ) {
    if ( !( ( $match_count == 2 ) && ( $matches[0] eq $matches[1] ) ) ) {
      error("cant recognize CPU_ALL ($match_count matches) in NMON file $file_to_proces");
      return ( "", "", 0 );
    }
  }
  ( undef, undef, my $rest ) = split( ",", $matches[0], 3 );
  if ( $rest !~ /^User%,Sys%,Wait%/ ) {
    error("cant recognize CPU_ALL in NMON line $matches[0] in file $file_to_proces");
    return ( "", "", 0 );
  }

  @matches = grep {/AAA,interval/} @part1;
  if ( @matches != 1 ) {
    error("cant recognize AAA,interval in NMON file $file_to_proces ");
    return ( "", "", 0 );
  }
  ( undef, undef, my $part ) = split( /,/, $matches[0] );
  $interval = $part;
  if ( $interval != ( $interval * 1 ) ) {
    error("cant recognize interval in NMON line $matches[0] in file $file_to_proces");
    return ( "", "", 0 );
  }

  # try to find out which system & lpar name

  my $is_aix = 0;
  @matches = grep { /AIX/ && !/lscfg/ } @part1;

  if ( @matches > 0 ) {    # working for AIX
    $is_aix = 1;

    $os_type = "AIX";

    $server  = "1165-J4K";                                     # fake name when there are no definition lines BBB
    @matches = grep { /System Model/ || /de syst/ } @part1;    #/System Model/ ||
    if ( @matches != 1 ) {

      # error ("cant recognize System Model: in NMON file $file_to_proces ");
      # return ("","",0);
    }
    else {
      ( undef, my $part ) = split( /:/, $matches[0] );
      ( undef, $server ) = split( /,/, $part );
      $server =~ s/\"//g;
      $server .= "*";
    }
    if ( $server =~ /1165-J4K/ ) {
      @matches = grep {/AAA,MachineType/} @part1;
      if ( @matches != 1 ) {
        error("cant recognize System Model: in NMON file $file_to_proces ");
        $is_aix = 0;
        # return ( "", "", 0 );
      }
      else {
        ( undef, undef, undef, $server ) = split( /,/, $matches[0] );
        $server =~ s/\"//g;
        $server .= "*";
      }
    }

    $part    = "00X0X0X";
    @matches = grep { /Machine Serial Number/ || /e de la machine/ } @part1;
    if ( @matches != 1 ) {

      # error ("cant recognize Machine Serial Number: in NMON file $file_to_proces ");
      # return ("","",0);
    }
    else {
      if ( defined $matches[0] and $matches[0] !~ 'Not Available' ) {
        ( undef, $part ) = split( /:/, $matches[0] );
        $part =~ s/ //g;
        $part =~ s/\"//g;
        $server .= $part;
      }
    }
    if ( $part eq "00X0X0X" ) {
      @matches = grep {/AAA,SerialNumber/} @part1;
      if ( @matches != 1 ) {
        error("cant recognize Machine Serial Number: in NMON file $file_to_proces ");
        $is_aix = 0;
        # return ( "", "", 0 );
      }
      else {
        ( undef, undef, $part ) = split( /,/, $matches[0] );
        $part =~ s/ //g;
        $part =~ s/\"//g;
        $server .= $part;
      }
    }

    if ( $yestime ne 'yestime' ) {
      $server =~ s/:/=====double-colon=====/g;
    }

    @matches = grep {/,lparname,/} @part1;
    if ( @matches != 1 ) {

      # error ("cant recognize lparname in NMON file $file_to_proces ");
    }
    else {
      ( undef, undef, undef, my $part ) = split( /,/, $matches[0] );
      $part =~ s/\"//g;
      $lpar = $part;
    }
    @matches = grep {/AAA,LPARNumberName/} @part1;
    if ( @matches != 1 ) {

      # error ("cant recognize AAA,LPARNumberName in NMON file $file_to_proces ");
    }
    else {
      ( undef, undef, $lpar_id, my $part ) = split( /,/, $matches[0] );
      $part =~ s/\"//g;
      $lpar_n = $part;
    }
    @matches = grep {/AAA,runname,/} @part1;
    if ( @matches != 1 ) {

      # error ("cant recognize AAA,runname, in NMON file $file_to_proces for $os_type");
    }
    else {
      ( undef, undef, my $part ) = split( /,/, $matches[0] );
      $part =~ s/\"//g;
      $lpar_runname = $part;
    }

    if ( length($lpar_n) > length($lpar) ) {
      $lpar = $lpar_n;
    }
    if ( length($lpar) == 0 ) {
      $lpar = $lpar_runname;
    }
    my $blade_lpar = "";
    if ( defined $ENV{LPAR2RRD_HOSTNAME_PREFER} && $ENV{LPAR2RRD_HOSTNAME_PREFER} == 1 ) {

      # special solution for blade server - lpar > prefare host name
      @matches = grep {/AAA,host,/} @part1;
      if ( @matches > 0 ) {
        ( undef, $blade_lpar ) = split( ',host,', $matches[0] );
      }
    }

    if ( $blade_lpar ne "" ) {
      $lpar = $blade_lpar;
    }

    if ( length($lpar) == 0 ) {
      error("no valid lpar-name in $file_to_proces ");
      $is_aix = 0;
      # return ( "", "", 0 );
    }

    if ( $yestime ne 'yestime' ) {
      $lpar =~ s/:/=====double-colon=====/g;
    }
  }
  if ($is_aix) {
    @matches = grep {/ZZZZ,T/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize ZZZ time in NMON file $file_to_proces ");
      return ( "", "", 0 );
    }
    my $time_stamp_unix = prepare_time( $matches[0] );

    if ( $yestime eq 'yestime' ) {
      return ( $server, $lpar, $time_stamp_unix );
    }

    @matches = grep {/Entitled processor capacity/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize Entitled processor capacity in NMON file $file_to_proces ");
    }
    else {
      ( undef, undef, undef, $part ) = split( /,/, $matches[0] );
      $part =~ s/\"ent_capacity//g;
      ( $part, undef ) = split( /Entitled/, $part );
      $part =~ s/ //g;
      $entitled_proc = $part;
    }

    @matches = grep {/Processor Clock Speed/} @part1;    #::
    if ( @matches != 1 ) {
      error("cant recognize Processor Clock Speed: in NMON file $file_to_proces ");
    }
    ( undef, $part ) = split( /:/, $matches[0] );
    $part =~ s/ //g;
    $part =~ s/\"//g;
    $part =~ s/MHz//g;
    $proc_clock = $part;

    @matches = grep { /Total Paging Space/ || /Espace de pagination total/ } @part1;    #::
    if ( @matches != 1 ) {
      print "cant recognize Total Paging Space: in NMON file $file_to_proces ";
    }
    else {
      ( undef, $part ) = split( /:/, $matches[0] );
      $part =~ s/ //g;
      $part =~ s/MB//g;
      $part =~ s/\"//g;
      $paging_space = $part;

      @matches = grep { /Percent Used/ || /Pourcentage utili/ } @part1;                 #::
      if ( @matches != 1 ) {
        error("cant recognize Percent Used: in NMON file $file_to_proces ");
      }
      ( undef, $part ) = split( /:/, $matches[0] );
      $part =~ s/ //g;
      $part =~ s/\%//g;
      $part =~ s/\"//g;
      $paging_percent = $part;
    }

    # NET,Network I/O asrv11lpar7,en2-read-KB/s,en3-read-KB/s,en4-read-KB/s,lo0-read-KB/s,en2-write-KB/s,en3-write-KB/s,en4-write-KB/s,lo0-write-KB/s
    # NETPACKET,Network Packets asrv11lpar7,en2-reads/s,en3-reads/s,en4-reads/s,lo0-reads/s,en2-writes/s,en3-writes/s,en4-writes/s,lo0-writes/s
    # NETSIZE,Network Size asrv11lpar7,en2-readsize,en3-readsize,en4-readsize,lo0-readsize,en2-writesize,en3-writesize,en4-writesize,lo0-writesize
    # NETERROR,Network Errors asrv11lpar7,en2-ierrs,en3-ierrs,en4-ierrs,lo0-ierrs,en2-oerrs,en3-oerrs,en4-oerrs,lo0-oerrs,en2-collisions,en3-collisions,en4-collisions,lo0-collisions

    prepare_net( "NET,Network", $file_to_proces );

    #FCREAD,Fibre Channel Read KB/s,fcs0,fcs1,fcs2,fcs3
    #FCWRITE,Fibre Channel Write KB/s,fcs0,fcs1,fcs2,fcs3
    #FCXFERIN,Fibre Channel Tranfers In/s,fcs0,fcs1,fcs2,fcs3
    #FCXFEROUT,Fibre Channel Tranfers Out/s,fcs0,fcs1,fcs2,fcs3

    #FCREAD,T0001,0.0,0.0,0.0,0.0
    #FCWRITE,T0001,0.0,0.0,0.0,0.0
    #FCXFERIN,T0001,0.0,0.0,0.0,0.0
    #FCXFEROUT,T0001,0.0,0.0,0.0,0.0

    prepare_sea_stats(@part1);
    prepare_fcread(@part1);

    # IOADAPT,T1440,0.0,0.0,0.0,0.0,0.0,0.0,1.1,43.2,10.2  9 cisel
    # IOADAPT,Disk Adapter sse11,fcs0_read-KB/s,fcs0_write-KB/s,fcs0_xfer-tps,fcs1_read-KB/s,fcs1_write-KB/s,fcs1_xfer-tps
    # IOADAPT,T0001,0.0,0.0,0.0,352.4,68.1,26.4            6 cisel
    # IOADAPT,Disk Adapter sse12,fcs1_read-KB/s,fcs1_write-KB/s,fcs1_xfer-tps,fcs0_read-KB/s,fcs0_write-KB/s,fcs0_xfer-tps
    #IOADAPT,Disk Adapter bsrv21lpar3,fcs0_read-KB/s,fcs0_write-KB/s,fcs0_xfer-tps,fcs1_read-KB/s,fcs1_write-KB/s,fcs1_xfer-tps
    #IOADAPT,Disk Adapter bsrv21lpar4,fcs1_read-KB/s,fcs1_write-KB/s,fcs1_xfer-tps,fcs0_read-KB/s,fcs0_write-KB/s,fcs0_xfer-tps
    #IOADAPT,Disk Adapter CBCFCBSDEVT,sissas1_read-KB/s,sissas1_write-KB/s,sissas1_xfer-tps,sissas0_read-KB/s,sissas0_write-KB/s,sissas0_xfer-tps

    @matches = grep {/IOADAPT,Disk/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize IOADAPT,Disk in NMON file $file_to_proces ");
    }
    else {
      my @linea = split( ",", $matches[0] );

      # print "\@linea (IOADAPT) @linea\n";
      # my @match_read = grep { /fcs[0-9]{1,2}_read/ } @linea;
      my @match_read = grep {/_read/} @linea;

      # print "match_read fcs @match_read\n";
      # my @match_write =  grep { /fcs[0-9]{1,2}_write/} @linea;
      my @match_write = grep {/_write/} @linea;
      if ( ( @match_read != @match_write ) || ( ( @match_read + @match_write ) < 2 ) || ( $match_read[0] =~ /not available/ ) ) {
        error("not same reads as writes (or zero) number in fcs line or not available");
      }
      else {
        # my @match_xfer = grep { /fcs[0-9]{1,2}_xfer/ } @linea;
        my @match_xfer = grep {/_xfer/} @linea;
        if ( @match_read != @match_xfer ) {
          error("not same reads as xfer number in fcs line");
        }
        for ( my $i = 0; $i < @match_read; $i++ ) {
          ( $fcs[$i], undef ) = split( "_", $match_read[$i] );
          ( $fcs_inx[ 4 * $i ] )     = grep { $linea[$_] =~ /$fcs[$i]/ } 0 .. $#linea;
          ( $fcs_inx[ 4 * $i + 1 ] ) = grep { $linea[$_] =~ /$fcs[$i]_write/ } 0 .. $#linea;
          ( $fcs_inx[ 4 * $i + 2 ] ) = grep { $linea[$_] =~ /$fcs[$i]_xfer/ } 0 .. $#linea;
          $fcs_inx[ 4 * $i + 3 ] = -1;
          print "$i, fcs @fcs fcs_inx @fcs_inx\n";
        }
      }
    }
    return ( $server, $lpar, $lpar_id, $paging_space, $paging_percent, $proc_clock );
  }

  @matches = grep {/Linux/} @part1;
  $os_type = "OS-like-Linux";

  if ( @matches > 0 ) {    # working for Linux - will you differ by flavours?

    @matches = grep {/Red Hat/} @part1;
    if ( @matches > 0 ) {
      $os_type = "LINUX-RedHat";
    }
    @matches = grep {/Arch Linux/} @part1;
    if ( @matches > 0 ) {
      $os_type = "LINUX-Arch";
    }

    @matches = grep {/Solaris/} @part1;
    foreach my $match (@matches) {

      # BBBP,001,/etc/release,"                             Oracle Solaris 11.2 X86"
      if ( index( $match, "etc/release" ) != -1 && index( $match, "Solaris" ) != -1 ) {
        $os_type = "UX-Solaris";
        last;
      }
    }

    @matches = grep {/Ubuntu/} @part1;
    if ( @matches > 0 ) {
      $os_type = "Ubuntu";
    }

    $server = $os_type;

    # testing Linux on Power
    # even for Linux you must test if Linux on Power
    # BBBP,692,/proc/ppc64/lparcfg,"lparcfg 1.9"
    # BBBP,693,/proc/ppc64/lparcfg,"serial_number=IBM,0244K8102"
    # BBBP,694,/proc/ppc64/lparcfg,"system_type=IBM,9117-MMC"
    # BBBP,695,/proc/ppc64/lparcfg,"partition_id=10"

    my $ibm_part_name = "";
    @matches = grep {/lparcfg,\"system_type=IBM,/} @part1;
    if ( @matches > 0 ) {
      ( undef, my $system_type ) = split /type=IBM,/, $matches[0];
      $system_type =~ s/ //g;
      $system_type =~ s/\"//g;
      chomp $system_type;
      $system_type .= "*";
      @matches = grep {/lparcfg,\"serial_number=IBM,/} @part1;
      if ( @matches > 0 ) {
        ( undef, my $serial_number ) = split /number=IBM,/, $matches[0];
        $serial_number =~ s/ //g;
        $serial_number =~ s/\"//g;
        chomp $serial_number;
        $serial_number = substr $serial_number, 2;
        $server        = $system_type . $serial_number;
        $print_line    = $print_line . " Linux on Power";
      }
      # find another partition name in possible
	  # BBBP,6725,lparstat -i,"Partition Name                               : b4chl22-ITR1_2"
	  @matches = grep {/BBBP,/ && /lparstat/ && /Partition Name/} @part1;
	  if ( @matches > 0 ) {
        ( undef, $ibm_part_name) = split /:/, $matches[0];
		chomp $ibm_part_name;
		$ibm_part_name =~ s/ //g;
		$ibm_part_name =~ s/\"//g;
		print "Linux on Power detected partition name:$ibm_part_name\n";
	  }
    }

    print "working for $os_type $print_line";

    @matches = grep {/AAA,host/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize AAA,host: in NMON file $file_to_proces ");
      return ( "", "", 0 );
    }
    ( undef, undef, my $part ) = split( /,/, $matches[0] );
    $part =~ s/ //g;
    my $lpar_n = $part;

    @matches = grep {/AAA,runname/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize AAA,runname in NMON file $file_to_proces ");
    }
    else {
      ( undef, undef, my $part ) = split( /,/, $matches[0] );
      $lpar = $part;
    }
    if ( length($lpar_n) > length($lpar) ) {
      $lpar = $lpar_n;
    }
    # for Linux on Power test if found partition name
	$lpar = $ibm_part_name if $ibm_part_name ne "";

    if ( $yestime ne 'yestime' ) {
      $lpar =~ s/:/=====double-colon=====/g;
    }

    @matches = grep {/ZZZZ,T/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize ZZZ time in NMON file $file_to_proces ");
      return ( "", "", 0 );
    }
    my $time_stamp_unix = prepare_time( $matches[0] );

    if ( $yestime eq 'yestime' ) {
      return ( $server, $lpar, $time_stamp_unix );
    }

    prepare_net( "NET,Network", $file_to_proces );

    return ( $server, $lpar, $lpar_id, "", "", "" );
  }

  error("cant recognize OS from NMON file $file_to_proces ");
  return ( "", "", 0 );
}    # end of sub read_first_part

sub prepare_fcread {
  my (@part1) = @_;

  my @matches = grep {/FCREAD,Fibre/} @part1;
  if ( @matches != 1 ) {

    #  error ("cant recognize FCREAD,Fibre in NMON file $file_to_proces ");
  }
  else {
    my @linea      = split( ",", $matches[0] );
    my @match_read = grep {/fcs[0-9]{1,2}/} @linea;
    print "match_read FCREAD fcs @match_read\n";

    @matches = grep {/FCWRITE,Fibre/} @part1;
    if ( @matches != 1 ) {
      error("cant recognize FCWRITE,Fibre after FCREADin NMON file ");
    }
    else {
      my @linea       = split( ",", $matches[0] );
      my @match_write = grep {/fcs[0-9]{1,2}/} @linea;
      print "match_write FCWRITE fcs @match_write\n";
      if ( ( @match_read != @match_write ) || ( ( @match_read + @match_write ) < 2 ) ) {
        error("not same FCREADs as FCWRITEs (or zero) number in fcs line");
      }
      for ( my $i = 0; $i < @match_read; $i++ ) {
        $fcf[$i] = $match_write[$i];
        ( $fcf_inx[$i] ) = grep { $linea[$_] =~ /$fcf[$i]/ } 0 .. $#linea;

        # ($fcf_inx[4*$i+1]) = grep { $linea[$_] =~ /$fcf[$i]/ } 0..$#linea;
        # ($fcf_inx[4*$i+2]) = grep { $linea[$_] =~ /$fcf[$i]/ } 0..$#linea;
        # $fcf_inx[4*$i+3] = grep { $linea[$_] =~ /$fcf[$i]/ } 0..$#linea;
        print "$i, fcf @fcf fcf_inx @fcf_inx\n";    #
      }
    }
  }
}

# BBBS,000,Shared Ethernet Adapter stats,SEAs found,9
# BBBS,001,SEA1,ent28
# BBBS,002,SEA1,ent28,Naccounting,ctl_chan,gvrp,ha_mode,jumbo_frames,large_receive,largesend,lldpsvc,netaddr,pvid,pvid_adapter,qos_mode,real_adapter,thread,virt_adapters
# BBBS,003,SEA1,ent28,disabled,ent18,no,auto,no,no,1,no,0,1,ent9,disabled,ent1,1,ent9
# BBBS,004,SEA2,ent29
# BBBS,005,SEA2,ent29,Naccounting,ctl_chan,gvrp,ha_mode,jumbo_frames,large_receive,largesend,lldpsvc,netaddr,pvid,pvid_adapter,qos_mode,real_adapter,thread,virt_adapters
# BBBS,006,SEA2,ent29,disabled,ent19,no,auto,no,no,1,no,0,2,ent10,disabled,ent2,1,ent10
# . . .
# SEA,Shared Ethernet Adapter bsrv22vios2,ent28-read-KB/s,ent29-read-KB/s,ent30-read-KB/s,ent31-read-KB/s,ent32-read-KB/s,ent33-read-KB/s,ent34-read-KB/s,ent35-read-KB/s,ent36-read-KB/s,ent28-write-KB/s,ent29-write-KB/s,ent30-write-KB/s,ent31-write-KB/s,ent32-write-KB/s,ent33-write-KB/s,ent34-write-KB/s,ent35-write-KB/s,ent36-write-KB/s
#
# SEA,T0002,0.6,0.6,3903.3,15.6,950.0,1781.8,2.0,16.2,0.4,0.6,0.6,3903.3,15.6,950.0,1781.8,1.9,16.2,0.7
#
#           $message .= ":sea:$back_en:$en:$transb:$recb\::::";
#

sub prepare_sea_stats {
  my (@part1) = @_;

  my @matches = grep {/Shared Ethernet Adapter stats/} @part1;
  print "SEA \@matches @matches\n";
  if ( scalar @matches == 0 ) {
    return;
  }
  ( undef, undef, undef, undef, my $eth_tot ) = split( ',', $matches[0] );

  # print "SEA after having \$eth_tot $eth_tot\n";
  if ( !isdigit($eth_tot) ) {
    error("cannot get number of SEA ethernet stats ,$eth_tot, or more than one stats");
    return;
  }
  @matches = grep {/BBBS,.*SEA.*ent[0-9]+,.*/} @part1;
  my $info_line_num = scalar @matches;
  if ( $info_line_num != ( 2 * $eth_tot ) ) {
    error("not 2 times ,$eth_tot, info lines for SEA!, it is ,$info_line_num,");
  }
  my @sorted_eth = sort @matches;

  # print "SEA \@sorted_eth @sorted_eth\n";
  my $text_line;
  my $value_line;
  while ( scalar(@sorted_eth) != 0 ) {
    $text_line = shift(@sorted_eth);
    if ( $text_line =~ /real_adapter/ ) {
      $value_line = shift(@sorted_eth);
    }
    else {
      $value_line = $text_line;
      $text_line  = shift(@sorted_eth);
    }
    my @name_array = split( ',', $text_line );
    my $index      = 0;
    ++$index until $name_array[$index] eq 'real_adapter' or $index > $#name_array;
    if ( $index > $#name_array ) {
      error("cannot find real_adapter item for SEA");
      return;
    }
    my @value_array = split( ',', $value_line );
    my $eth_value   = $value_array[$index];
    my $eth_to_push = "$name_array[3],$eth_value";
    push @phys_log_eth_sea, $eth_to_push;
  }
  print "@phys_log_eth_sea\n";

  #  analyse data info line - it is one line, prepare table for read_one_run
  #  SEA,Shared Ethernet Adapter bsrv22vios2,
  #  ent28-read-KB/s,ent29-read-KB/s,ent30-read-KB/s,ent31-read-KB/s,ent32-read-KB/s,
  #  ent33-read-KB/s,ent34-read-KB/s,ent35-read-KB/s,ent36-read-KB/s,
  #  ent28-write-KB/s,ent29-write-KB/s,ent30-write-KB/s,ent31-write-KB/s,ent32-write-KB/s,
  #  ent33-write-KB/s,ent34-write-KB/s,ent35-write-KB/s,ent36-write-KB/s

  @matches = grep {/SEA,Shared Ethernet Adapter /} @part1;
  if ( @matches != 1 ) {

    # error ("cant recognize SEA,Shared Ethernet Adapter in NMON file $file_to_proces ");
  }
  else {
    my @linea      = split( ",", $matches[0] );
    my @match_read = grep {/ent[0-9]{1,2}-read/} @linea;
    print "match_read SEA ent @match_read\n";

    my @match_write = grep {/ent[0-9]{1,2}-write/} @linea;
    print "match_write SEA ent @match_write\n";
    if ( ( @match_read != @match_write ) || ( ( @match_read + @match_write ) < 2 ) ) {
      error("not same SEA reads as writes (or zero) number in SEA line");
    }
    for ( my $i = 0; $i < @match_read; $i++ ) {
      ( my $et_name, undef ) = split( "-", $match_write[$i] );    # no matter read or write
      my ($phy_log_inx) = grep { $phys_log_eth_sea[$_] =~ /$et_name/ } 0 .. $#phys_log_eth_sea;
      my $phy_log_item = $phys_log_eth_sea[$phy_log_inx];
      print "found item phy_log ,$phy_log_item,\n";
      ( undef, my $phys_name ) = split( ',', $phy_log_item );

      #print "\$et_name $et_name $match_write[$i]\n";
      my ($read_inx)  = grep { $linea[$_] =~ /$et_name-read/ } 0 .. $#linea;    #index
      my ($write_inx) = grep { $linea[$_] =~ /$et_name-write/ } 0 .. $#linea;
      $sea[$i] = "$et_name XORUX $read_inx XORUX $write_inx XORUX $phys_name";
      print "$i, SEA @sea \n";
    }
  }
}    # end of sub prepare_sea_stats

sub prepare_time {

  my $line = shift;

  require Time::Local;

  # example:  ZZZZ,T1440,hh:mm:ss,27-MAR-2014  prepare timestamp

  ( undef, $line ) = split( /ZZZ,/, $line );    # if not in start of line

  # print "3328 line $line\n";

  ( undef, my $time_stamp_text, my $date_day, undef ) = split( /,/, $line );
  my %mon2num = qw(
    jan 0  feb 1  mar 2  apr 3  may 4  jun 5
    jul 6  aug 7  sep 8  oct 9  nov 10  dec 11
  );
  ( my $day, my $month, my $year ) = split( '-', $date_day );
  if ( $day < 1 || $day > 31 ) { $day   = 1 }
  if ( length $month != 3 )    { $month = "JAN" }
  if ( $year < 2000 )          { $year  = 2000 }
  my $monum = $mon2num{ lc substr( $month, 0, 3 ) };
  ( my $hourd, my $mind, my $secd ) = split( ':', $time_stamp_text );
  return Time::Local::timelocal( $secd, $mind, $hourd, $day, $monum, $year );
}

sub prepare_net {
  my $test           = shift;
  my $file_to_proces = shift;

  my @matches = grep {/$test/} @part1;
  if ( @matches != 1 ) {
    error("cant recognize $test in NMON file $file_to_proces ");
  }
  else {
    my @linea      = split( ",", $matches[0] );
    my @match_read = grep {/[A-Za-z]+[0-9]{1,3}-read/} @linea;
    print "match_read en @match_read\n";
    my @match_write = grep {/[A-Za-z]+[0-9]{1,3}-write/} @linea;
    if ( @match_read != @match_write ) {
      error("not same reads as writes number in line $matches[0]");
    }
    if ( @match_read == 0 ) {
      print "zero reads number in en $test line $matches[0] in file $file_to_proces\n";
    }
    else {
      for ( my $i = 0; $i < @match_read; $i++ ) {
        ( $ent[$i], undef ) = split( "-", $match_read[$i] );
        ( $ent_inx[ 4 * $i ] ) = grep { $linea[$_] =~ /$ent[$i]-read/ } 0 .. $#linea;
        ( $ent_inx[ 4 * $i + 1 ] ) = grep { $linea[$_] =~ /$ent[$i]-write/ } 0 .. $#linea;
        $ent_inx[ 4 * $i + 2 ] = -1;    # same look as fcs
        $ent_inx[ 4 * $i + 3 ] = -1;
        print "$i, en @ent ent_inx @ent_inx\n";
      }
    }
  }

}

sub read_one_run {
  my $start_time     = shift;
  my $file_to_proces = shift;

  #  my $DEBUG = 0;
  #     $DEBUG = 3;
  my $mem                 = "";
  my $memnew              = "";
  my $page                = "";
  my $lpar_data           = "";
  my $lpar_data_cpu       = "";
  my $net_data            = "";
  my $net_packet          = "";
  my $fcsnet_data         = "";
  my $seanet_data         = "";
  my $san_disk_read_data  = "";
  my $san_disk_write_data = "";
  my $san_iops_data       = "";
  my $san_resp_data       = "";
  my $san_wait_data       = "";
  # PAGE,Paging sse21,faults,pgin,pgout,pgsin,pgsout,reclaims,scans,cycles
  my $zzzz            = "";
  my $line            = "";
  my $indx            = 0;
  my $time_stamp_unix = 0;

  if ( $zzzz_this !~ "ZZZZ,T" ) {
    print "didnt find ZZZZ,T for read_one_run, probably end of file $file_to_proces\n$last_zzzz_this";
    return 0;
  }

  #  print " date $zzzz_this\n";

  $time_stamp_unix = prepare_time($zzzz_this);

  my $act_time    = localtime($time_stamp_unix);
  my @arr_one_run = ();

  while ( ( defined( $line = <DF_IN> ) ) && ( $line !~ "ZZZZ,T" ) ) {
    $line =~ s/\r[\n]*/\n/gm;    # all platforms Unix, Windows, Mac
    chomp($line);
    push @arr_one_run, $line;    # read lines from one run
  }
  if ( !defined($line) ) {
    $zzzz_this = "";
  }
  else {
    $zzzz_this      = $line;
    $last_zzzz_this = $line;
  }

  if ( $time_stamp_unix <= $start_time ) {    #already sent
    return $start_time;
  }

  # working out memory characteristics
  # AIX
  # MEM,Memory,Real Free %,Virtual free %,Real free(MB),Virtual free(MB),Real total(MB),Virtual total(MB)

  my ( $free, $size, $inuse ) = ( 0, 0, 0 );
  my $pin         = 0;
  my $in_use_clnt = 0;
  my $in_use_work = 0;
  my ( $user_proc, $free_proc );

  my $test_item = "MEM,";
  my @matches   = grep {/^$test_item/} @arr_one_run;
  my $count_m   = @matches;
  if ( $count_m ne 1 ) {
    error("$count_m items for $test_item in \@arr_one_run not possible")

      #     return $time_stamp_unix;
  }
  if ( $count_m == 1 ) {
    $mem = $matches[0];
    if ( $os_type eq "AIX" ) {
      ( undef, undef, undef, undef, $free, undef, $size, undef ) = split( /,/, $mem );
      $size *= 1024;
      $free *= 1024;
      $inuse = $size - $free;

      $test_item = "MEMNEW,";
      @matches   = grep {/^$test_item/} @arr_one_run;
      $count_m   = @matches;
      if ( $count_m > 1 ) {
        error("$count_m items for $test_item in \@arr_one_run not possible");
      }
      if ( $count_m == 0 ) {
        $memnew = "";
      }
      else {
        $memnew = $matches[0];

        #MEMNEW,Memory New sse21,Process%,FScache%,System%,Free%,Pinned%,User%

        if ( $memnew ne "" ) {
          ( undef, undef, my $proces, my $in_use_clnt_proc, my $syst_proc, $free_proc, $pin, $user_proc ) = split( /,/, $memnew );
          $in_use_clnt = $in_use_clnt_proc * $size / 100;
          $pin         = $pin * $size / 100;
          $in_use_work = ( $proces + $syst_proc ) * $size / 100;
          @matches     = grep {/^MEMAMS/} @arr_one_run;
          if ( @matches == 1 ) {
            $inuse = ( $syst_proc + $user_proc ) * $size / 100 + $in_use_clnt;
            $free  = $size - $inuse;
            if ( ( ( $user_proc + $free_proc + $syst_proc ) * $size / 100 ) > ( $size - $in_use_clnt ) ) {
              $free  = $free_proc * $size / 100;
              $inuse = $size - $free;
            }
          }
        }
      }
    }

    # Red Hat or Arch
    # MEM,Memory MB bsrpmgt0007,memtotal,hightotal,lowtotal,swaptotal,memfree,highfree,lowfree,
    #                           swapfree,memshared,cached,active,bigfree,buffers,swapcached,inactive
    # Total= memtotal
    # FS cache = cached
    # Free = memfres
    # Mem used = memtotal – memfree

    if ( $os_type =~ "LINUX-RedHat" || $os_type =~ "LINUX-Arch" || $os_type =~ "Ubuntu" || $os_type =~ "OS-like-Linux" ) {
      ( undef, undef, $size, undef, undef, undef, $free, undef, undef, undef, undef, $in_use_clnt, undef ) = split( /,/, $mem );
      $size        *= 1024;
      $free        *= 1024;
      $in_use_clnt *= 1024;    # FS cache
      $inuse = $size - $free;

      #print "\$size $size \$free $free \$in_use_clnt $in_use_clnt, \$inuse $inuse\n $mem\n";
    }
                               #
                               #Solaris
                               #
    if ( $os_type eq "UX-Solaris" ) {

      #  MEM,Memory MB bsrpdev0035,memtotal,NA1,NA2,swaptotal,memfree,NA3,NA4,swapfree,swapused
      ( undef, undef, $size, undef, undef, $paging_space, $free, undef, undef, undef, my $val_a ) = split( /,/, $mem );
      $size *= 1024;
      $free *= 1024;

      #$in_use_clnt *= 1024; # FS cache
      $inuse = $size - $free;
      my $paging_percenta = "";
      if ( $paging_space == 0 ) {
        $paging_percenta = $paging_space;
      }
      else {
        $paging_percenta = $val_a / $paging_space;
      }
      $paging_percent = $paging_percenta * 100;

      #print "\$size $size \$free $free \$in_use_clnt $in_use_clnt, \$inuse $inuse\n $mem\n";
    }
  }

  $test_item = "PAGE,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  if ( @matches == 0 ) {
    $page = "";
  }
  else {
    $page = $matches[0];
  }

  $test_item = "LPAR,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  if ( @matches == 1 ) {
    $lpar_data = $matches[0];
  }
  else {
    $lpar_data = "";
  }

  $test_item = "CPU_ALL,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  $lpar_data_cpu = $matches[0];

  $test_item = "NET,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  $net_data = $matches[0];

  $test_item = "NETPACKET,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  $net_packet = $matches[0];

  $test_item = "IOADAPT,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  $fcsnet_data = $matches[0];

  $test_item = "SEA,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  if ( @matches > 1 ) {
    error("More items $test_item in \@arr_one_run not possible");
  }
  $seanet_data = $matches[0];

  #  print "$line\n"  if $DEBUG == 3;

  #### SOLARIS SAN
  #DISKREAD,Disk Read kb/s sol10,sd0,sd1
  #DISKWRITE,Disk Write kb/s sol10,sd0,sd1
  #DISKXFER,Disk Transfers per second sol10,sd0,sd1
  #DISKSVCTM,Disk Svc Time ms sol10,sd0,sd1
  #DISKWAITTM,Disk Wait Time ms sol10,sd0,sd1
  my $san_message = "";

  if ( $os_type =~ /Solaris/ ) {
    $test_item = "DISKREAD,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    $san_disk_read_data = $matches[0];

    my $san_disk_read = 0;
    if (defined $san_disk_read_data && $san_disk_read_data ne ""){
      ( undef,undef, my @san_data ) = split /,/, $san_disk_read_data;
      if (@san_data) {
        my $number_of_disk = @san_data;
        foreach my $san_disk (@san_data){
          $san_disk_read += $san_disk;
        }
      }
      #print "SAN_DISK_READ(@san_data):$san_disk_read\n";
    }

    $test_item = "DISKWRITE,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    $san_disk_write_data = $matches[0];

    my $san_disk_write = 0;
    if (defined $san_disk_write_data && $san_disk_write_data ne ""){
      ( undef,undef, my @san_data ) = split /,/, $san_disk_write_data;
      if (@san_data) {
        my $number_of_disk = @san_data;
        foreach my $san_disk (@san_data){
          $san_disk_write += $san_disk;
        }
      }
      #print "SAN_DISK_WRITE(@san_data):$san_disk_write\n";
    }

    $test_item = "DISKXFER,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    $san_iops_data = $matches[0];

    my $san_iops = 0;
    if (defined $san_iops_data && $san_iops_data ne ""){
      ( undef,undef, my @san_data ) = split /,/, $san_iops_data;
      if (@san_data) {
        my $number_of_disk = @san_data;
        foreach my $san_disk (@san_data){
          $san_iops += $san_disk;
        }
      }
      #print "SAN_IOPS(@san_data):$san_iops\n";
    }

    $test_item = "DISKSVCTM,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    $san_resp_data = $matches[0];

    $test_item = "DISKWAITTM,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    $san_wait_data = $matches[0];

    my $san_latency = 0;
    if (defined $san_resp_data && $san_resp_data ne "" && defined $san_wait_data && $san_wait_data ne ""){
      ( undef,undef, my @san_data1 ) = split /,/, $san_resp_data;
      ( undef,undef, my @san_data2 ) = split /,/, $san_wait_data;
      my $san_resp = 0;
      if (@san_data1) {
        my $number_of_disk = @san_data1;
        foreach my $san_disk (@san_data1){
          $san_resp += $san_disk;
        }
      }
      my $san_wait = 0;
      if (@san_data2) {
        my $number_of_disk = @san_data2;
        foreach my $san_disk (@san_data2){
          $san_wait += $san_disk;
        }
      }
      $san_latency = $san_wait + $san_resp;
      #print "SAN_LATENCY:(@san_data1 | @san_data2):$san_latency\n";
    }
      $san_message .= ":sanmon:$san_disk_read:$san_disk_write:$san_iops:$san_latency\::::";
    #print "SAN DISK READ:  $san_disk_read_data\nSAN DISK WRITE: $san_disk_write_data\nSAN IOPS:  $san_iops_data \nSAN RESP:  $san_resp_data\nSAN WAIT: $san_wait_data\n";
  }

  # PAGE,Paging sse21,faults,pgin,pgout,pgsin,pgsout,reclaims,scans,cycles

  # 39 items
  # MEMPAGES4KB,MemoryPages,numframes,numfrb,numclient,numcompress,numperm,numvpages,
  # minfree,maxfree,numpout,numremote,numwseguse,numpseguse,numclseguse,numwsegpin,
  # numpsegpin,numclsegpin,numpgsp_pgs,numralloc,pfrsvdblks,pfavail,pfpinavail,numpermio,
  # system_pgs,nonsys_pgs,pgexct,pgrclm,pageins,pageouts,
  #
  # pgspgins,pgspgouts, # this is interesting items 31 and 32
  #
  # numsios,numiodone,zerofills,exfills,scans,cycles,pgsteals

  # same for MEMPAGES64KB

  # since 4.69
  # page_in = pgsin * interval / 4.096     translate to 4 kB blocks
  #  my $page_const = $interval/4.096;
  #  no it is the same as pi and po in vmstat - 4k blocks / sec
  #  my $page_const = 1;
  #
  # Solaris data
  # VM,Paging and Virtual Memory up102ora0101c,minfaults,majfaults,pgin,pgout,scans,reclaims,pgpgin,pgpgout,pswpin,pswpout,pgfree
  # VM,T1244,5360.8,4.2,4.2,0.0,0.0,0.0,33.9,0.0,0.0,0.0,0.0
  # pgpgin
  # pgpgout    (KB/s) pages paged in and out
  # Same as 'vmstat.pi and po'

  my $page_in  = 0;
  my $page_out = 0;

  if ( $os_type eq "UX-Solaris" ) {
    $test_item = "VM,";
    @matches   = grep {/^$test_item/} @arr_one_run;
    if ( @matches > 1 ) {
      error("More items $test_item in \@arr_one_run not possible");
    }
    ( undef, undef, undef, undef, undef, undef, undef, undef, $page_in, $page_out ) = split( /,/, $matches[0] );

  }
  else {
    if ( $page ne "" ) {
      $test_item = "MEMPAGES64KB,";
      @matches   = grep {/^$test_item/} @arr_one_run;
      if ( @matches > 0 ) {    # means there is 64 KB
        my @arr    = split( /,/, $matches[0] );
        my $pgsin  = $arr[30] * 16;
        my $pgsout = $arr[31] * 16;
        $test_item = "MEMPAGES4KB,";
        @matches   = grep {/^$test_item/} @arr_one_run;
        if ( @matches > 0 ) {    # means there is 4 KB
          @arr = split( /,/, $matches[0] );
          $pgsin  += $arr[30];
          $pgsout += $arr[31];
        }
        $pgsin  /= $interval;
        $pgsout /= $interval;
        $page_in  = $pgsin;
        $page_out = $pgsout;

        # print "yes 64 $pgsin $pgsout\n";
      }
      else {
        ( undef, undef, undef, undef, undef, my $pgsin, my $pgsout ) = split( /,/, $page );
        $page_in  = $pgsin;
        $page_out = $pgsout;

        #print "no 64 $pgsin $pgsout\n";
      }
    }
  }

  # prepare cpu
  # LPAR,Logical Partition sse21,PhysicalCPU,virtualCPUs,logicalCPUs,poolCPUs,entitled,weight,PoolIdle,usedAllCPU%,usedPoolCPU%,SharedCPU,Capped,EC_User%,EC_Sys%,EC_Wait%,EC_Idle%,VP_User%,VP_Sys%,VP_Wait%,VP_Idle%,Folded,Pool_id
  # LPAR,T1440,0.100,3,12,15,0.30,128,11.66,0.63,0.67,1,0,14.16,7.34,0.00,11.90,1.42,0.73,0.00,1.19,0
  # for both types of lpars entitled_proc in first_part
  # BBBP,536,lsattr -El sys0,"ent_capacity    1.00    Entitled processor capacity   False"

  my $lpar_message = "";
  my $cpu_us       = "";
  my $cpu_sy       = 0;
  my $cpu_wa       = 0;
  my $entitled     = 0;
  my $physical_cpu = 0;

  # CPU_ALL,T1440,10.4,7.4,0.1,82.2,,12
  # These lines show %usr, %sys, %wait and %idle by time of day for logical processors

  #test if message contains CPU & MEM
  if ( ( !defined $lpar_data_cpu ) && $lpar_data eq "" && $count_m != 1 ) {
    print "not complete message left out before $zzzz_this";
    return $time_stamp_unix;
  }

  $entitled = $entitled_proc if defined($entitled_proc);
  if ( ( !defined $lpar_data_cpu ) && $lpar_data eq "" ) {
    error("no info for CPU_ALL in file $file_to_proces");
    $lpar_message = ":lpar:::$entitled:U\::::";
  }
  else {
    if ( $lpar_data_cpu ne "" ) {
      ( undef, undef, $cpu_us, $cpu_sy, $cpu_wa ) = split( /,/, $lpar_data_cpu );
    }
    if ( $lpar_data ne "" ) {
      if ( $cpu_us eq "" ) {
        ( undef, undef, $physical_cpu, undef, undef, undef, $entitled, undef, undef, undef, undef, undef, undef, $cpu_us, $cpu_sy, $cpu_wa ) = split( /,/, $lpar_data );
      }
      else {
        ( undef, undef, $physical_cpu, undef, undef, undef, $entitled ) = split( /,/, $lpar_data );
      }
      $lpar_message = ":lpar:::$entitled:$physical_cpu\::::";
    }
  }

  # prepare ethernet adapter
  # NET,Network I/O sse21,en2-read-KB/s,en4-read-KB/s,lo0-read-KB/s,en2-write-KB/s,en4-write-KB/s,lo0-write-KB/s
  # NET,T1440,0.1,1.2,0.0,0.0,0.5,0.0

  my $lan_message = "";

  if ( defined $ent[0] && defined $net_data && $net_data ne "" ) {

    # print "net_data, $net_data \$net_packet $net_packet\n"; # if $DEBUG == 3;
    my @temparr        = split( /,/, $net_data );
    my @temparr_packet = split( /,/, $net_packet ) if $net_packet ne "";
    my $adapters       = @ent;

    # print "number of items in net_data, $adapters ".@temparr."\n" ;
    for ( my $i = 0; $i < $adapters; $i++ ) {
      next if ( $ent[$i] =~ /^lo/ );
      my $read      = $temparr[ $ent_inx[ 4 * $i ] ] * 1024;
      my $write     = $temparr[ $ent_inx[ 4 * $i + 1 ] ] * 1024;
      my $xfer_read = $temparr[ $ent_inx[ 4 * $i + 2 ] ];
      if ( $ent_inx[ 4 * $i + 2 ] == -1 ) {
        if ( $net_packet ne "" ) {
          $xfer_read = $temparr_packet[ $ent_inx[ 4 * $i ] ];
        }
        else {
          $xfer_read = 0;
        }
      }
      my $xfer_write = $temparr[ $ent_inx[ 4 * $i + 3 ] ];
      if ( $ent_inx[ 4 * $i + 3 ] == -1 ) {
        if ( $net_packet ne "" ) {
          $xfer_write = $temparr_packet[ $ent_inx[ 4 * $i + 1 ] ];
        }
        else {
          $xfer_write = 0;
        }
      }
      $lan_message .= ":lan:$ent[$i]:nmonip:$write:$read:$xfer_write:$xfer_read\::";

      # print "\$lan_message $lan_message\n";
    }
  }

  # prepare fcsi adapter see sub read_first_part
  # FCREAD,T0001,0.0
  # FCWRITE,T0001,0.0
  # FCXFERIN,T0001,0.0
  # FCXFEROUT,T0001,0.0

  prepare_fcread(@arr_one_run);    #definition part can be also in data part

  my $fclan_message = "";
  if ( defined $fcf[0] ) {
    my @match_fcread    = grep {/^FCREAD,T/} @arr_one_run;
    my @match_fcwrite   = grep {/^FCWRITE,T/} @arr_one_run;
    my @match_fcxferin  = grep {/^FCXFERIN,T/} @arr_one_run;
    my @match_fcxferout = grep {/^FCXFEROUT,T/} @arr_one_run;
    my $fc_sum          = @match_fcread + @match_fcwrite + @match_fcxferin + @match_fcxferout;

    if ( ( $fc_sum > 3 ) && ( ( $fc_sum % 4 ) == 0 ) ) {
      my @fcread_arr    = split( /,/, $match_fcread[0] );
      my @fcwrite_arr   = split( /,/, $match_fcwrite[0] );
      my @fcxferin_arr  = split( /,/, $match_fcxferin[0] );
      my @fcxferout_arr = split( /,/, $match_fcxferout[0] );

      my $fc_full = "full";    # signal for daemon to touch signal for detail-graph-cgi.pl to graph xin/xout separately
      for ( my $i = 0; $i < @fcf; $i++ ) {
        my $read       = $fcread_arr[ $fcf_inx[$i] ] * 1024;
        my $write      = $fcwrite_arr[ $fcf_inx[$i] ] * 1024;
        my $xfer_read  = $fcxferin_arr[ $fcf_inx[$i] ] + $fcxferout_arr[ $fcf_inx[$i] ];
        my $xfer_write = $fcxferout_arr[ $fcf_inx[$i] ];
        $fclan_message .= ":san:$fcf[$i]:nmonip:$read:$write:$xfer_read:$xfer_write:$fc_full\:";
      }
    }
  }

  # in case defined FCREAD WRITE do not do this
  # prepare fcsi adapter see sub read_first_part
  # @fcs - fcs names eg fcs0, fcs1
  # @fcs_inx - pointers to fcs data line for fcs-read & write

  if ( ( defined $fcs[0] ) && ( !defined $fcf[0] ) && ( defined $fcsnet_data ) ) {
    $fclan_message = "";
    if ( $fcsnet_data ne "" ) {

      # print "fcsnet_data, $fcsnet_data\n" if $DEBUG == 3;
      my @temparr  = split( /,/, $fcsnet_data );
      my $adapters = @fcs;

      # print "number of items in fcs net_data, $adapters ".@temparr."\n" if $DEBUG == 3;
      for ( my $i = 0; $i < $adapters; $i++ ) {
        my $read       = $temparr[ $fcs_inx[ 4 * $i ] ] * 1024;
        my $write      = $temparr[ $fcs_inx[ 4 * $i + 1 ] ] * 1024;
        my $xfer_read  = $temparr[ $fcs_inx[ 4 * $i + 2 ] ];
        my $xfer_write = $temparr[ $fcs_inx[ 4 * $i + 3 ] ];
        $xfer_write = 0 if $fcs_inx[ 4 * $i + 3 ] == -1;
        if ( $fcs[$i] !~ /available/ ) {
          $fclan_message .= ":san:$fcs[$i]:nmonip:$read:$write:$xfer_read:$xfer_write\::";
        }
      }
    }
  }

  # prepare SEA adapter see sub read_first_part
  # @sea array names (starting 0) ent28-write-KB/s ent29-write-KB/s ent30-write-KB/s ...
  # @sea_inx array (starting 0) 11 12 13 14 15 16 17 18 19 ...
  # @phys_log_eth_sea   ent28,ent1 ent29,ent2 ent30,ent3 ent31,ent4 ... actually log,phys

  my $sea_message = "";
  if ( ( defined $sea[0] ) && ( defined $seanet_data ) ) {
    if ( $seanet_data ne "" ) {

      # print "seanet_data, $seanet_data\n";
      my @temparr  = split( /,/, $seanet_data );
      my $adapters = @sea;

      # print "number of items in sea net_data, $adapters and data items ".@temparr."\n";
      for ( my $i = 0; $i < $adapters; $i++ ) {
        ( my $et_name, my $read_inx, my $write_inx, my $phys_name ) = split( ' XORUX ', $sea[$i] );
        my $read  = int( $temparr[$read_inx] * 1024 );
        my $write = int( $temparr[$write_inx] * 1024 );
        if ( $sea[$i] !~ /available/ ) {
          $sea_message .= ":sea:$phys_name:$et_name:$write:$read\::::";
        }
      }
    }

    # print "\$sea_message $sea_message\n";
  }

  # Processes
  # AIX https://www.ibm.com/docs/en/aix/7.2?topic=tool-kernel-statistics
  # PROC,Processes bsrv22vios2,Runnable,Swap-in,pswitch,syscall,read,write,fork,exec,sem,msg,asleep_bufio,asleep_rawio,asleep_diocio
  # PROC,T0003,0.63,0.00,731,468,45,5,2,3,0,0,0,0,0
  # Run Queue is sum of the number of processes that are currently running and the number of processes that are waiting (queued) to run.
  # Pswitch is the Number of context switches rates per second
  # Syscall is the total number of system calls per second. A value of -1 means that number is not supported.
  # asleep_bufio  Number of processes that are in sleep state and are waiting for buffered I/O operation to complete at a particular time.
  # asleep_rawio  Number of processes that are sleeping and waiting for raw I/O operations at a particular time. Raw I/O operation allows applications to perform a direct write operation to the Logical Volume Manager (LVM) layer.
  # asleep_diocio Number of processes that are in sleep state and are waiting for direct file system I/O operations and concurrent I/O operations at a particular time.
  #
  # LINUX
  # PROC,Processes vm-jindra,Runnable,Blocked,pswitch,syscall,read,write,fork,exec,sem,msg
  # PROC,T0002,1,0,222.9,-1.0,-1.0,-1.0,3.1,-1.0,-1.0,-1.0
  #
  # SOLARIS
  # PROC,Processes vm-solaris,Runnable,Swap-in,pswitch,syscall,read,write,fork,exec,sem,msg,%RunOcc,%SwpOcc,kthrR,kthrB,kthrW
  # PROC,T0002,1.1,0.0,183.5,293.2,8.4,1.3,0.6,0.8,0.0,0.0,16.4,0.0,0.2,0.0,0.0
  #
  # number of active cores
  # Aix
  # AAA,cpus,16,16    all, active
  #
  # Linux
  # AAA,cpus,4
  # AAA,x86,VirtualCPUs,4
  #
  # Solaris
  # AAA,cpus,2,2    all, active
  #
  my $runnable     = "";
  my $swap_in      = "";
  my $pswitch      = "";
  my $aaacpus      = "";
  my $proc_message = "";

  $test_item = "PROC,";
  @matches   = grep {/^$test_item/} @arr_one_run;
  $count_m   = @matches;
  if ( $count_m ne 1 ) {
    error("$count_m items for $test_item in \@arr_one_run not possible")

      # return $time_stamp_unix;
  }

  # print STDERR "6078 $matches[0]\n";
  # error("6078 $matches[0]\n");
  if ( $count_m == 1 ) {
    my $proc = $matches[0];
    if ( $os_type eq "AIX" ) {

      # old AIX PROC,Processes bsux0287,Runnable,Swap-in,pswitch,syscall,read,write,fork,exec,sem,msg
      # new AIX PROC,Processes bsrv22vios2,Runnable,Swap-in,pswitch,syscall,read,write,fork,exec,sem,msg,asleep_bufio,asleep_rawio,asleep_diocio
      ( undef, undef, $runnable, $swap_in, $pswitch, undef, undef, undef, undef, undef, undef, undef, undef, my $rawio, my $diocio, undef ) = split( /,/, $proc );
      $rawio  = "" if !defined $rawio;
      $diocio = "" if !defined $diocio;

      # error("6091 :$rawio:$diocio:\n");
      # CPU_ALL,T0009,66.6,12.4,5.6,15.4,,64
      $test_item = "CPU_ALL,";
      @matches   = grep {/^$test_item/} @arr_one_run;
      if ( scalar @matches == 1 ) {
        $aaacpus = ( split ",", $matches[0] )[-1];
        if ( $swap_in eq "" ) { $swap_in = "0"; }
        $proc_message = ":queue_cpu_aix:::$runnable:$aaacpus:$swap_in\:$rawio:$diocio:";
      }
    }
        #return ":queue_cpu_aix:::$load_parsed:$cpu_cnt[0]:$data[$blocked_indx]:$data[$raw_indx]:$data[$direct_io_indx]:";
    if ( $os_type =~ "LINUX-RedHat" || $os_type =~ "LINUX-Arch" || $os_type =~ "Ubuntu" || $os_type =~ "OS-like-Linux" || $os_type eq "UX-Solaris" ) {

      # if ( $os_type =~ "LINUX-RedHat" || $os_type =~ "LINUX-Arch" || $os_type =~ "Ubuntu" || $os_type =~ "OS-like-Linux") {
      ( undef, undef, $runnable, $swap_in, $pswitch, undef ) = split( /,/, $proc );

      # $test_item = "AAA,cpus,"; #this must be from the 1st part
      # CPU_ALL,CPU Total vm-jindra,User%,Sys%,Wait%,Idle%,Busy,CPUs

      $test_item = "CPU_ALL,";
      @matches   = grep {/^$test_item/} @arr_one_run;
      $count_m   = @matches;
      if ( scalar @matches == 1 ) {
        $aaacpus = ( split ",", $matches[0] )[-1];
        if ( $swap_in eq "" ) { $swap_in = "0"; }
        $proc_message = ":queue_cpu:::$runnable:$aaacpus:$swap_in\:::";

        # print "6109 \$proc $proc \$proc_message ,$proc_message,\n";
      }
    }
  }

  if ( $cpu_us eq "" ) { $cpu_us = 0 }

  $message = "$server:$lpar:$lpar_id:$time_stamp_unix:$act_time version $version:$proc_clock\:$interval:$nmon_external_time:N";

  #  if (($size + $inuse + $free + $pin + $in_use_work + $in_use_clnt) != 0) left_curly
  $message .= ":mem:::$size:$inuse:$free:$pin:$in_use_work:$in_use_clnt";

  #  }

  $message .= ":pgs:::$page_in:$page_out:$paging_space:$paging_percent\::" if $paging_space ne "";

  #  if (($cpu_sy + $cpu_us + $cpu_wa) > 0) {
  $message .= ":cpu:::$entitled:$cpu_sy:$cpu_us:$cpu_wa\::";

  #  }
  if ( $os_type =~ /Solaris/ ) {
    $message .= "$san_message";
  }
  $message .= "$lan_message";
  $message .= "$fclan_message";
  $message .= "$sea_message";
  #$message .= "$san_message";

  if ( index( $nmon_data, "ext_nmon_" ) != -1 ) {    # external NMON
    $message .= "$lpar_message";
  }

  $message .= "$proc_message";

   print "6150 \$os_type $os_type $message\n";

  #   testing slash in server name
  # if ($message =~ "Red") {
  #   $message =~ s/Red/R\/ed/;
  # }

  print FFA "$message\n";

  return $time_stamp_unix;
}

sub process_limits {
  my ( $cmd, $limit, $ps_all_tmp ) = @_;
  my @ps_all   = @{$ps_all_tmp};
  my $ps_count = 0;

  foreach my $ps_line (@ps_all) {
    if ( $ps_line =~ m/$cmd/ && $ps_line !~ m/grep/ ) {

      #print "$ps_line";
      $ps_count++;
    }
  }

  if ( $ps_count > $limit ) {
    return 1;
  }
  return 0;
}

sub my_handler {
  print STDERR "The OS agent has crashed on signal [$!], no core on purpose\n";
  if ( -f $iostat_data_file ) {
    unlink($iostat_data_file);    # temporary iostat file
  }
  if ( -f $nfs_iostat_data_file ) {
    unlink($nfs_iostat_data_file);
  }
  if ( -f $zonestat_data_file ) {
    unlink($zonestat_data_file);
  }
  if ( -f $prstat_data_file ) {
    unlink($prstat_data_file);
  }
  if ( -f $wlmstat_data_file ) {
    unlink($wlmstat_data_file);
  }
  if ( -f $poolstat_data_file ) {
    unlink($poolstat_data_file);
  }
  exit(1);
}

#Clean error log if it is too big (1MB usually)
sub error_log_wipe {
  my $file     = shift;
  my $max_size = shift;
  if ( -e "$file" ) {
    if ( -r "$file" ) {

      #  print "file can read \n";
      my $size = ( stat $file )[7];
      if ( $size <= $max_size ) {

        #  print "file has not been deleted\n";
      }
      else {
        print "Cleaning the error log: $file ($size>=$max_size)\n";
        unlink($file) or print "file $file could not been deleted $!\n";
      }
    }
  }
  return 1;
}

sub get_smt {
  my ($amepat)    = @_;
  my @amepat_info = @{$amepat};
  my $smt         = 0;

  ($smt) = grep {/^SMT Threads/} @amepat_info;
  if ( defined $smt ) {
  $smt =~ s/^SMT Threads//g;
  $smt =~ s/\://g;
  $smt =~ s/^\s+//g;
  $smt =~ s/\s+$//g;
  }
  else {
    return "";
  }

  if ( $smt != 0 && $smt != 2 && $smt != 4 && $smt != 8 && $smt != 16 && $smt != 32 && $smt != 64 ) {
    return "";
  }

  return $smt;

}

sub ishexa {
  my $digit = shift;

  if ( !defined($digit) || $digit eq '' ) {
    return 0;
  }

  my $digit_work = $digit;
  $digit_work =~ s/^0x//;
  $digit_work =~ s/^0X//;
  $digit_work =~ s/[0-9]//g;
  $digit_work =~ s/\.//;
  $digit_work =~ s/^-//;
  $digit_work =~ s/e//;
  $digit_work =~ s/\+//;

  # hexa filter
  $digit_work =~ s/[a|b|c|d|e|f|A|B|C|D|E|F]//g;

  if ( length($digit_work) == 0 ) {

    # is a number
    return 1;
  }

  return 0;
}

sub read_ps_aeo {
  my $os           = shift;
  my $process_args = shift;
  my $result_ref   = shift;

  #test OS-
  if ( $os =~ m/Linux/ ) {
    @$result_ref = `ps -aeo pid,user,time,vsize,rssize,$process_args`;
  }
  elsif ( $os =~ m/SunOS/ ) {

    # Solaris does not support change args to command! Therefore the last argument is fixed here
    @$result_ref = `ps -aeo pid,user,time,vsz,rss,args`;
  }
  elsif ( $os =~ m/AIX/ ) {

    #print "PS-JOB:::This is AIX\n";
    @$result_ref = `ps -aeo pid,user,time,vsize,rssize,args`;

    #my $size_job=scalar @ps_job_old;-
    #print "PS-JOB-ARRAY:::@ps_job_old\n";
  }
  else {
    print "PS-JOB: UNKNOWN OS\n";
  }
}

sub write_ps_aeo {
  my $ps_job_config = shift;
  my $time_write    = shift;
  my $time_unix     = shift;
  my $ps_job_ref    = shift;

  #write new ps to file
  open( my $CF, "> $ps_job_config" ) or print( "PS-JOB: Cannot write $ps_job_config: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  print $CF "$time_write UNIX:$time_unix \n";
  foreach (@$ps_job_ref) {
    my $line_old = $_;

    #example:'  230 00:00:00 114644   680 root     /usr/bin/rsync --daemon --no-detach'
    chomp $line_old;
    $line_old =~ s/^\s+//;    #remove leading spaces
    ( my $pid, my $user, my $time_2,, my $vzs, my $rss, my $command ) = split( / +/, $line_old, 6 );

    # attn: comma not possible
    $command =~ s/,/---comma---/g;
    print $CF "$pid,$user,$command,$time_2,$vzs,$rss\n";
  }
  close($CF);

  return 0;
}

sub prepare_job_time {
  my $time_1 = shift;

  my $time_1_sec = 0;

  # prepare unix time from 'ps' time format
  my $count_time_1 = ( $time_1 =~ tr/:// );
  if ( $count_time_1 == 0 ) { return $time_1_sec }
  ;    # some trash

  # print "2112 \$count_time_1 $count_time_1 \$command $command\n";
  if ( $count_time_1 == "1" ) {
    ( my $min_new, my $sec_new ) = split( /:/, $time_1 );
    $time_1_sec = $min_new * 60 + $sec_new;
  }
  elsif ( $count_time_1 == "2" ) {

    # print "2118 \$count_time_1 $count_time_1 \$command $command\n";
    my $count_dash_1 = grep( /-/, $time_1 );

    #print "$count_dash_1\n";
    if ( $count_dash_1 == "0" ) {
      ( my $hour_new, my $min_new, my $sec_new ) = split( /:/, $time_1 );
      $time_1_sec = $hour_new * 3600 + $min_new * 60 + $sec_new;
    }
    else {
      ( my $hour_dash_1, my $min_new, my $sec_new ) = split( /:/, $time_1 );
      ( my $days_new, my $hour_new ) = split( /-/, $hour_dash_1 );
      $time_1_sec = $days_new * 86400 + $hour_new * 3600 + $min_new * 60 + $sec_new;
    }
  }
  else {
    print "PS-JOB: UNKNOWN TIME FORMAT\n";
  }
  return $time_1_sec;
}

sub ps_job {

  my %cpu_top    = ();
  my $time_write = $act_time;
  my $time_unix  = $act_time_u;
  my $time_1_sec;
  my $time_2_sec;
  my @ps_job_new;
  my @ps_job_old;
  my $ps_job_config = "$base_dir/lpar2rrd-agent-$host-$user_name-ps_job.txt";
  my $time_test;
  my $raw_time;
  my $command_out;
  my $time_delay          = 1800;      # basic interval for jobs to scan
  my $time_diff_limit     = 2;         # take care only jobs with at least this run-time
  my $time_diff_limit_min = 1;         # when nothing above time_diff_limit then use this limit to send at least 2 jobs always
  my $MAX_JOBS            = 50;        # number or transfered jobs
  my $process_include     = "";        # it watch only selected processes, regex is possible
  my $process_args        = "args";    # default ps -aeo pid,user,time,vsize,rssize,args can be changed to ps -aeo pid,user,time,vsize,rssize,command

  # read parameters from cfg file
  my $cfg_file = "/opt/lpar2rrd-agent/lpar2rrd-agent.cfg";
  if ( -f $cfg_file ) {
    open( FCFG, "< $cfg_file" ) || error( "Cannot read $cfg_file: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
    my @cfg = <FCFG>;
    close(FCFG);
    foreach my $line (@cfg) {
      chomp($line);
      if ( $line =~ m/^#/ ) {
        next;
      }
      if ( $line =~ m/^JOB_TOP=/ ) {
        my ( undef, $number ) = split( /=/, $line );
        $number =~ s/ //g;
        $number =~ s/#.*$//g;    #remove comment
        if ( !($number eq '') ) {
          if ( $number =~ m/no/ || $number =~ m/NO/ ) {
            print "JOB TOP setting: JOB_TOP=$number\n";
            return "";           # skip JOB TOP
          }
        }
      }
      if ( $line =~ m/^MAX_JOBS=/ ) {
        my ( undef, $number ) = split( /=/, $line );
        $number =~ s/ //g;
        $number =~ s/#.*$//g;    #remove comment
        if ( isdigit($number) ) {
          $MAX_JOBS = $number;
        }
      }
      if ( $line =~ m/^LOAD_LIMIT=/ ) {
        my ( undef, $number ) = split( /=/, $line );
        $number =~ s/ //g;
        $number =~ s/#.*$//g;    #remove comment
        if ( isdigit($number) ) {
          $time_diff_limit = $number;
        }
      }
      if ( $line =~ m/^LOAD_LIMIT_MIN=/ ) {
        my ( undef, $number ) = split( /=/, $line );
        $number =~ s/ //g;
        $number =~ s/#.*$//g;    #remove comment
        if ( isdigit($number) ) {
          $time_diff_limit_min = $number;
        }
      }
      if ( $line =~ m/^TIME_DELAY=/ ) {
        my ( undef, $number ) = split( /=/, $line );
        $number =~ s/ //g;
        $number =~ s/#.*$//g;    #remove comment
        if ( isdigit($number) ) {
          $time_delay = $number;
        }
      }
      if ( $line =~ m/^PROCESSES_INCLUDE=/ ) {
        ( undef, $process_include ) = split( /=/, $line );
        $process_include =~ s/#.*$//g;    #remove comment
        $process_include =~ s/ *$//;      #remove spaces
      }
      if ( $line =~ m/^PROCCESS_ARGS=/ ) {
        ( undef, $process_args ) = split( /=/, $line );
        $process_args =~ s/#.*$//g;       #remove comment
        $process_args =~ s/ *$//;         #remove spaces
        $process_args =~ s/"//g;
      }
    }
  }
  print "JOB TOP setting: MAX_JOBS=$MAX_JOBS, LOAD_LIMIT=$time_diff_limit, LOAD_LIMIT_MIN=$time_diff_limit_min, PROCESSES_INCLUDE=$process_include PROCCESS_ARGS=$process_args\n";

  return if $time_unix % $time_delay > 59;    # continue only in 1st minute, also in ($time_delay/60)-th minute
                                              # exact time when 1st minute starts, sometimes it is here 1 sec late
  $time_unix = $time_unix - ( $time_unix % $time_delay );

  #print "PS-JOB-OS:::$os\n";

  # do not use older file than twice $time_delay as there can be high peak in graphs
  if ( ( $time_unix - ( stat($ps_job_config) )[9] ) > ( $time_delay * 2 ) ) {
    unlink $ps_job_config;
  }

  if ( !-f $ps_job_config ) {
    read_ps_aeo( $os, $process_args, \@ps_job_old );
    write_ps_aeo( $ps_job_config, $time_write, $time_unix, \@ps_job_old );
  }

  #
  # Job TOP END
  #

  open( FC, "< $ps_job_config" ) || error( "Cannot read $ps_job_config: $!" . __FILE__ . ":" . __LINE__ ) && return "";
  @ps_job_old = <FC>;
  close(FC);

  ( undef, my $unix_time_old ) = split( /UNIX:/, $ps_job_old[0] );
  chomp $unix_time_old;
  my $time_test_diff_2 = $time_unix - $unix_time_old;

  if ( $time_test_diff_2 >= $time_delay ) {
    unlink $ps_job_config;
  }

  #print "PS_JOB-end\n";
  #print "PS_JOB_timetest1:::$time_test_1_sec\n";
  #print "PS_JOB_timetest2:::$time_test_2_sec\n";
  #print "PS_JOB_timetestdiff_end:::$time_test_diff_1\n";
  #print STDERR "-- 00 lest start\n";

  if ( $time_test_diff_2 >= $time_delay ) {    #if time difference > $time_delay min write new PS JOB
    read_ps_aeo( $os, $process_args, \@ps_job_new );

    #create time difference & save to hash-
    foreach my $line_new (@ps_job_new) {
      chomp $line_new;

      # print "4050 PS_JOB_LINEnew:::$line_new\n";
      $line_new =~ s/^\s+//;                   #remove leading spaces
      ( my $pid, my $user, my $time_1, my $vzs, my $rss, my $command ) = split( / +/, $line_new, 6 );
      next if not( isdigit($pid) );

      #print STDERR "-- 01 $process_include - $command : $line_new\n";
      if ( !($process_include eq '') ) {

        # select only defined processes
        if ( $command !~ m/$process_include/ ) {
          next;
        }
      }

      #print STDERR "-- 02 $process_include - $command : $line_new\n";

      # print "4054 PS_JOB_LINEnew:::\$time_1 $time_1 $line_new\n";

      $time_1_sec = prepare_job_time($time_1);

      my @line_old;

      #print "PSJOB_OLD:::@ps_job_old\n";
      #print "PSJOB PID:::\$pid $pid \$user $user\n";
      $command =~ s/,/---comma---/g;

      #print "PSJOB COMMAND:::$command\n";
      @line_old = grep {/^\Q$pid,$user,$command,\E/} @ps_job_old;

      # print "2141 PS_JOB-oldline_1:::@line_old, ,$pid,$user,$command,\n";
      if ( scalar @line_old > 1 ) {
        print "PS-JOB: More than 1 jobs";
      }
      next if ( !defined $line_old[0] );

      # print "2146 PS_JOB-oldline_1:::@line_old\n";
      ( undef, undef, undef, my $time_2, undef ) = split( /,/, $line_old[0] );

      # print "4095 PS_JOB_TIME2:::$time_2\n";

      $time_2_sec = prepare_job_time($time_2);

      my $time_difference = $time_1_sec - $time_2_sec;

      # print "2174 PS_JOB:::$time_1_sec\n";
      # print "PS_JOB/DIFF:::$time_difference,",",$command\n";
      next if ( $time_difference < $time_diff_limit_min );    #if time difference is less than $time_diff_limit_min do not work with it
                                                              #print STDERR "-- 03 $process_include - $command : $line_new\n";
      $command_out = $command;
      $command_out =~ s/://g;
      $command_out =~ s/---comma---/,/g;

      # $command_out = substr( $command_out, 0, 90 );

      #print "PS_JOB_outcommand:::$command_out\n";
      my $output = ":CPUTOP:$pid:$user:$command_out:$time_1_sec:$time_difference:$rss:$vzs\:";

      # print "2181 PS_JOB/OUTPUT:::$output\n";
      #print "2184 into hash \$output $output \$time_difference $time_difference\n";
      $cpu_top{$output} = $time_difference;
    }

    #print TOP10 JOB
    my $count_pr      = "0";
    my $ps_job_output = "";

    #print "print HASH\n";>-
    #print Dumper \%cpu_top;
    foreach my $key ( sort { $cpu_top{$b} <=> $cpu_top{$a} } keys %cpu_top ) {
      my ( undef, undef, undef, undef, undef, undef, $time_diff ) = split( /:/, $key );

      #print STDERR "-- 0 $count_pr $key --- $time_diff\n";
      if ( !isdigit($time_diff) ) {
        next;
      }
      if ( $count_pr < $MAX_JOBS && $time_diff > $time_diff_limit ) {
        $ps_job_output .= $key;
        $count_pr++;
        if ( $count_pr == $MAX_JOBS ) {
          last;
        }
      }
    }

    if ( $count_pr == 0 ) {

      # no job higher than limit is there, place there at least 2 to have some graphs
      $MAX_JOBS = 2;
      foreach my $key ( sort { $cpu_top{$b} <=> $cpu_top{$a} } keys %cpu_top ) {
        my ( undef, undef, undef, undef, undef, undef, $time_diff ) = split( /:/, $key );

        #print STDERR "-- 1 $count_pr $key --- \n";
        if ( !isdigit($time_diff) ) {
          next;
        }
        if ( $count_pr < $MAX_JOBS && $time_diff > $time_diff_limit_min ) {
          $ps_job_output .= $key;
          $count_pr++;
          if ( $count_pr == $MAX_JOBS ) {
            last;
          }
        }
      }
    }

    #write new ps to file
    my $ret = write_ps_aeo( $ps_job_config, $time_write, $time_unix, \@ps_job_new );
    if ( $ret == 1 ) {
      return "";
    }

    return $ps_job_output;
  }
}

sub trimlog {
  my $file  = shift;
  my $limit = shift;

  my $check_limit = $limit + ( $limit * 0.1 );
  my $wcs         = `wc -l $file 2>/dev/null`;

  if (-f $file) {
    my ( $size, $filename ) = split ' ', $wcs;
    if ( $filename ne "total" && $size > $check_limit ) {
      print "trim logs      : file $filename contains $size lines, it will be trimmed to $limit lines\n";
      error("trim logs      : file $filename contains $size lines, it will be trimmed to $limit lines");

      my $keepfrom     = $size - $limit;
      my $filename_tmp = "$filename-tmp";

      open( IN,  "< $filename" )     || error( "Couldn't open file $filename $!" . __FILE__ . ":" . __LINE__ )     && exit;
      open( OUT, "> $filename_tmp" ) || error( "Couldn't open file $filename_tmp $!" . __FILE__ . ":" . __LINE__ ) && exit;

      my $count = 0;
      while ( my $iline = <IN> ) {
        chomp $iline;
        $count++;
        if ( $count > $keepfrom ) {
          print OUT "$iline\n";
        }
      }
      close IN;
      close OUT;

      rename "$filename_tmp", "$filename";
    }
  }

  return 1;
}

sub file_time_diff {
  my $file = shift;

  my $act_time  = time();
  my $file_time = $act_time;
  my $time_diff = 0;

  if ( -f $file ) {
    $file_time = ( stat($file) )[9];
    $time_diff = $act_time - $file_time;
  }

  return ($time_diff);
}

sub wlmstat {

  my $wlmstat_data_file = shift;
  my $WLMSTAT           = shift;
  my $WLMSTAT_M         = shift;
  my $DEBUG             = 1;
  my $class             = "";
  my $cpu               = "";
  my $mem               = "";
  my $dkio              = "";
  my $class_type        = "";
  my $data              = "";

  print "$WLMSTAT 2>/dev/null\n";      # must be to /dev/null otherwise it produces in case no WLM: 1495-104  WLM must be started.
  print "$WLMSTAT_M 2>/dev/null\n";    # it does not exist on AIX 5.1 therefore /dev/null
  my $wlmstat_cmd     = "";
  my $wlmstat_MEM_cmd = "";

  if ( process_limits( "$WLMSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$WLMSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    $wlmstat_cmd = `$WLMSTAT 2>/dev/null`;    # it is in pages, # must be to /dev/null otherwise it produces in case no WLM: 1495-104  WLM must be started.
  }

  if ( process_limits( "$WLMSTAT_M", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$WLMSTAT_M\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    $wlmstat_MEM_cmd = `$WLMSTAT_M 2>/dev/null`;    # it is in pages, it does not exist on AIX 5.1 therefore /dev/null
  }

  #$data .= "$os\:$user_name\:$act_time_u\:$act_time version\: $version\:|";
  if ( !defined $wlmstat_cmd ) {
    print STDERR "Output of command: $WLMSTAT was empty.\n" if $DEBUG == 3;
  }
  if ( !defined $wlmstat_MEM_cmd ) {
    print STDERR "Output of command: $WLMSTAT was empty.\n" if $DEBUG == 3;
  }
  print "Output from \"$WLMSTAT\" command: $wlmstat_cmd \n" if $DEBUG == 3;

  my @split     = split( /\n/, $wlmstat_cmd );
  my @split_MEM = split( /\n/, $wlmstat_MEM_cmd );

  my $count = @split;
  for ( my $i = 1; $i < $count; $i++ ) {    #First line == head => $i = 1;
                                            #($class, undef, undef, undef $cpu, undef,undef undef,undef,undef undef,undef,undef undef,undef,undef undef,undef,undef undef,undef,undef undef, $dkio) = split ' ', $split[$i];
    ( $class, $cpu, undef, $dkio ) = split ' ', $split[$i];
    ( undef, $mem, undef, undef, undef, undef, undef ) = split ' ', $split_MEM[$i];
    $class =~ s/^\s+|\s+$//g;               #removes whitespaces
    $cpu   =~ s/^\s+|\s+$//g;
    $mem   =~ s/^\s+|\s+$//g;
    $dkio  =~ s/^\s+|\s+$//g;
    my $dot_count = () = $class =~ /[.]/g;
    my $test      = $i - $count;             # For checking last item

    if ( !isdigit($cpu) || $cpu eq "-" ) {   #test if given parametrs are numbers
                                             #error("wlmstats:      CPU of Class: $class is not a digit $cpu\n");
      $cpu = "U";
    }
    elsif ( $cpu > 100 || $cpu < 0 ) {       #test if usage is not greater than 100% or lower than 0
                                             #error("wlmstats:      CPU of Class: $class has usage $cpu\%");
      $cpu = "U";
    }

    if ( !isdigit($mem) || $mem eq "-" ) {

      #error("wlmstats:      MEM of Class: $class is not a digit $mem\n");
      $mem = "U";
    }

    if ( !isdigit($dkio) || $dkio eq "-" ) {

      #error("wlmstats:      DKIO of Class: $class is not a digit $dkio\n");
      $dkio = "U";
    }
    elsif ( $dkio > 100 || $dkio < 0 ) {

      #error("wlmstats:      DKIO of Class: $class has usage $dkio\%");
      $dkio = "U";
    }
    if ( $cpu eq "U" && $mem eq "U" && $dkio eq "U" ) {
      error("wlmstats:      $class has not been sent: all values empty\n");
      next;
    }
    if ( $dot_count == 0 ) {
      $class_type = "sup";
    }
    else {
      $class_type = "sub";
    }
    if ( $class eq "TOTAL" ) {
      $data .= "$class-$cpu-$mem-$dkio-$class_type";
    }
    else {
      $data .= "$class-$cpu-$mem-$dkio-$class_type\n";
    }
  }

  open( WLM, ">> $wlmstat_data_file" ) || error( "Cannot open $wlmstat_data_file\: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  print WLM "$data\n";
  close(WLM);
  return 1;
}

sub prstat {
  my $prstat_data_file = shift;
  my $PRSTAT           = shift;
  print "$PRSTAT\n";
  my @prstat_cmd = "";
  if ( process_limits( "$PRSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$PRSTAT\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @prstat_cmd = `$PRSTAT 2>>$error_log`;
  }
  my @zone_list_sol10 = "";
  if ( process_limits( "$ZONEADM", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error( "there is already running $PROC_RUN_LIMIT copies of \"$ZONEADM\", skipping it " . __FILE__ . ":" . __LINE__ );
  }
  else {
    @zone_list_sol10 = `$ZONEADM 2>>$error_log`;
  }
  my @smbios    = "";
  my $uuid_name = "";
  my $zone_name = "";
  my $zone_grep = "";
  if ( $type_of_solaris =~ /sparc/ ) {

    #print "$SNEEP\n";
    if ( process_limits( "$SNEEP", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$SNEEP\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      #  @smbios = `$SNEEP 2>>$error_log`;
    }
  }
  else {
    if ( process_limits( "$SMBIOS", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
      error( "there is already running $PROC_RUN_LIMIT copies of \"$SMBIOS\", skipping it " . __FILE__ . ":" . __LINE__ );
    }
    else {
      my $zonename = `zonename`;
      chomp $zonename;
      if ( $zonename eq "global" ) {    #### command SMBIOS not run on zone
        @smbios = `$SMBIOS 2>>$error_log`;
      }
    }
  }

  open( PRS, ">> $prstat_data_file" ) || error( "Cannot open $prstat_data_file: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  if ( $type_of_solaris =~ /sparc/ ) {
    $uuid_name = "@smbios";
    chomp $uuid_name;
  }
  else {
    ($uuid_name) = grep ( /UUID:/, @smbios );
  }
  $uuid_name =~ s/UUID://g;
  $uuid_name =~ s/\s+//g;
  print "@zone_list_sol10\n";
  foreach my $zone_sol (@zone_list_sol10) {
    if ( $zone_sol=~ /ID/ && $zone_sol =~ /NAME/ ) { next; }
    chomp $zone_sol;
    ( undef, undef, $zone_name ) = split( /\s+/, $zone_sol );
    ($zone_grep) = grep ( /% $zone_name/, @prstat_cmd );
    if ($zone_grep) {
      my $prstat = $zone_grep;
      my ( undef, $zone_id, $nproc, $swap, $rss, $memory, $time, $cpu, $zone_name ) = ( split( /\s+/, $prstat ) );
      chomp( $zone_id, $nproc, $swap, $rss, $memory, $time, $cpu, $zone_name );
      if ( defined $zone_id && $zone_id eq "" ) {next}
      my @ldom_info4 = "";
      if ( $type_of_solaris =~ /sparc/ ) {
        if ( process_limits( "$VIRTINFO", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
          error( "there is already running $PROC_RUN_LIMIT copies of \"$VIRTINFO\", skipping it " . __FILE__ . ":" . __LINE__ ) && next;
        }
        else {
          @ldom_info4 = `$VIRTINFO 2>>$error_log`;
        }
      }
      my $ldom_uuid_zone = "";
      my $char_l         = "";
      my $zone_name1     = `zonename 2>/dev/null`;
      chomp $zone_name1;
      if ( $zone_name1 eq "global" ) {
        if ( grep /Domain UUID:/, @ldom_info4 ) {
          ($ldom_uuid_zone) = grep /Domain UUID:/, @ldom_info4;
          $ldom_uuid_zone =~ s/Domain UUID://g;
          $ldom_uuid_zone =~ s/\s+//g;
          $char_l = "/";
          chomp $ldom_uuid_zone;
        }
        $swap   =~ s/M//;    ### MB
        $rss    =~ s/M//;    ### MB
        $memory =~ s/%//;    ### percent
        $cpu    =~ s/%//;    ### percent
                             #$message .= ":$zone_name$char_l$ldom_uuid_zone$char_l$ldom_uuid_zone_name:$uuid_name:\:$zone_id:$memory:$cpu:\:\:";
        print PRS":$zone_name$char_l$ldom_uuid_zone:$uuid_name:\:$zone_id:$memory:$cpu:\:\:\n";
      }
    }
  }
  close(PRS);
  return 1;
}

sub zonestat {
  my $zonestat_data_file = shift;
  my $ZONESTAT           = shift;

  if ( process_limits( "$ZONESTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error("there is already running $PROC_RUN_LIMIT copies of \"$ZONESTAT\"");
    return 1;
  }

  open my $ZON, ">", $zonestat_data_file
    or error("Cannot open $zonestat_data_file: $!") && return 1;

  my @out = `$ZONESTAT 2>>$error_log`;
  print $ZON @out;

  close $ZON;
  return 1;
}

sub poolstat {
  my $poolstat_data_file = shift;
  my $POOLSTAT           = shift;

  if ( process_limits( "$POOLSTAT", $PROC_RUN_LIMIT, \@ps ) == 1 ) {
    error("there is already running $PROC_RUN_LIMIT copies of \"$POOLSTAT\"");
    return 1;
  }

  open my $POOL, ">", $poolstat_data_file
    or error("Cannot open $poolstat_data_file: $!") && return 1;

  my @out = `$POOLSTAT 2>>$error_log`;
  print $POOL @out;

  close $POOL;
  return 1;
}

sub translate_df_output {

  # argument: OS, i.e., format of the `df` command/output: "aix" or by default "gnu"
  # my $platform = @_ ? shift : "gnu";
  my $DF_AIX7  = shift;
  my $DF_AIX   = shift;
  my $DF_SOL   = shift;
  my $DF_LIN   = shift;
  my $platform = shift;

  # command: disk free w/ 1K block size, skip remote filesystems if possible
  my $command = "";    #( $platform eq "AIX" ) ? "$DF_AIX 2>>$error_log" : "$DF_LIN_SOL 2>>$error_log";

  if ( $platform =~ m/AIX/ ) { $command = "$DF_AIX7 2>/dev/null" }
  elsif ( $platform =~ m/Linux/ ) {
    my $dfcmd = `which df`;
    chomp $dfcmd;
    my $target = readlink($dfcmd);
    if ( $target && $target =~ "busybox" ) {
      $DF_LIN = "df -kP";
    }
    $command = "$DF_LIN 2>>$error_log";
  }
  elsif ( $platform =~ m/SunOS/ ) { $command = "$DF_SOL 2>/dev/null" }    # lpar2rrd user does not have permission to all zone filesystems
  my @rows = `$command`;
  if ( !@rows && $platform =~ m/AIX/ ) {

    # Older AIXes
    $command = "$DF_AIX 2>/dev/null";
    @rows    = `$command`;
  }
  print "FS: $command \n";

  my ( @columns, $fs, $mount_path, $blocks_total, $blocks_used, $blocks_free, $used_ratio, $output );
  foreach my $row (@rows) {
    chomp $row;

    if ( $row =~ /^Filesystem/ ) {
      next;
    }

    if ( $row =~ /\// ) {

      # select just lines where is "/"

      $row =~ tr/ //s;
      @columns = split / /, $row;

      if ( $platform =~ m/SunOS/ ) {
        if ( $columns[0] =~ /^\/cdrom$/ || $columns[5] =~ /^\/cdrom\// ) {
          # skip cdrom 
          next;
        }
      }

      if ( $platform =~ m/Linux/ ) {
        if ( $columns[0] =~ /^\/dev\/sr[0-9]$/ || $columns[0] =~ /^\/dev\/cdrom/ || $row =~ /^efivarfs/ || $row =~ /\/media\/CDROM/ || $row =~ /^overlay / || $row =~ /^shm / || $row =~ /^cgmfs / || $columns[0] =~ /^\/dev\/loop[0-9]/ || $columns[0] =~ /^\/snap\// || $columns[0] =~ /^\/run\// || $columns[5] =~ /^\/sys\// || $columns[5] =~ /^\/swap/ || $columns[5] =~ /^\/run\// || $columns[5] =~ /^\/dev\// ) {
          # ^tmpfs is ok, there can be /tmp for example, some of ^tmpfs are excluded by ^\/dev\/ , ^\/run\/ 8.20-4
          # /boot also enabled 8.20-4
          # skip cdrom on Linux
          next;
        }
      }

      # Filesystem              1K-blocks      Used Available Use% Mounted on

      if ( $platform eq "AIX" ) {
        if ( $columns[5] =~ /\/wpars\// or $columns[0] =~ /^\/proc/ or $columns[0] =~ /:\// ) {

          # skip the row, if it corresponds to an AIX workload partition (container) or /proc or NFS mounted fs (includes ":/" in the name)
          print "FS: skipping as AIX workload partition (container) or /proc or NFS mounted fs: $row \n";
          next;
        }
      }

      if ( grep { $_ eq '-' } @columns[ 1 .. 4 ] ) {

        # skip the row, if the device doesn't have capacity in some way
        print "FS: skipping as node \"-\" in size/used/free appeared: $row \n";
        next;
      }

      if ( $columns[1] == 0 ) {

        #skip if 1K blocks are 0
        print "FS: skipping as size == 0: $row \n";
        next;
      }

      if ( $columns[5] =~ m/^\/dev$/ || $columns[5] =~ m/^\/dev\/.*/ ) {

        # skip if mount starts with /dev
        print "FS: skipping as mount starts with /dev: $row \n";
        next;
      }

      $mount_path   = $columns[5];
      $fs           = $columns[0];
      $blocks_total = units_transfer( $columns[1] );
      $blocks_used  = units_transfer( $columns[2] );
      $blocks_free  = units_transfer( $columns[3] );
      $used_ratio   = substr( $columns[4], 0, -1 );

      $mount_path =~ s/:/=====double-colon=====/g;
      $fs         =~ s/:/=====double-colon=====/g;

      $output .= ":";
      $output .= join ':', ( 'FS', $mount_path, $fs, $blocks_total, $blocks_used, $blocks_free, $used_ratio );
      $output .= "::";

    }
    else {
      # either output header
      # or a temporary/remote filesystem
      # ---PH : Solaris ZFS might not have to "/" in each line (total pool lines) ... it should be included
      print "FS: skipping as Solaris ZFS might not have to / in each line (total pool lines): $row \n";
      next;
    }
  }

  open( FS, "> $fs_storae_file" ) || error( "Cannot open $fs_storae_file\: $!" . __FILE__ . ":" . __LINE__ ) && return 1;
  print FS "$output\n";
  close(FS);

  return $output;
}

sub units_transfer {

  my $number = shift;
  my $divide = 1048576;

  my $output = $number / $divide;
  $output = sprintf( "%.2f", $output );
  return $output;

}

sub load_hitachi_uuid_file {
  my %lpar_uuids;
  my $hvmsh_host     = $opt_i;
  my $hvmsh_api_path = $opt_b;
  my $uuid_file      = "/var/tmp/hitachi-$hvmsh_host-lpar2rrd.uuids";

  if ( -e $uuid_file && ( time() - ( stat($uuid_file) )[9] < 3600 ) ) {    ## read from existing uuids file
    open( UUID, "< $uuid_file" ) || error( "Cannot read $uuid_file: $!" . __FILE__ . ":" . __LINE__ );
    while ( my $line = <UUID> ) {
      chomp $line;
      my ( $name, $uuid ) = split /,/, $line;
      $lpar_uuids{$name} = $uuid;
    }
    close(UUID);
  }
  else {                                                                   ## update old or create new uuids file
    open( UUID, "> $uuid_file" ) || error( "Can't open $uuid_file: $!" . __FILE__ . ":" . __LINE__ ) && return \%lpar_uuids;

    my $config = `$hvmsh_api_path -host=$hvmsh_host get ConfigAll 2>>$error_log`;
    $config =~ /\[LPAR_CONFIGURATION\]\n.*\n([^\[]*)\[/;
    my $LPAR_cfg = $1;

    for my $entry ( split /\n/, $LPAR_cfg ) {
      my @row = split /\t+/, $entry;
      my ( $name, $uuid ) = @row[ 2, 22 ];

      unless ( $name eq "*" || $uuid eq "*" ) {
        $lpar_uuids{$name} = $uuid;
        print UUID "$name,$uuid\n";
      }
    }

    close(UUID);
  }
  return \%lpar_uuids;
}

sub data_collect_hvmsh {
  my $act_time_u      = shift;
  my $act_time        = shift;
  my $message         = "";
  my $HVMSH_RUN_LIMIT = 100;
  my $hvmsh_tmp_file  = "/var/tmp/hitachi-$hvmsh_host-tmp.txt";
  my %lpar_uuids      = %{ load_hitachi_uuid_file() };

  unless ( -e $hvmsh_api_path ) {
    error( "can't locate HvmSh api file " . __FILE__ . ":" . __LINE__ );
    return "";
  }

  unless ( -e $hvmsh_tmp_file ) {
    my $ret = `touch $hvmsh_tmp_file`;

    if ($ret) {
      error( "$ret " . __FILE__ . ":" . __LINE__ );
    }
  }

  if ( process_limits( "HvmSh", $HVMSH_RUN_LIMIT, \@ps ) ) {
    error( "there is already running $HVMSH_RUN_LIMIT copies of \"HvmSh\", skipping it " . __FILE__ . ":" . __LINE__ );
    return "";
  }

  my $perfmon = `$hvmsh_api_path -host=$hvmsh_host get HvmPerfMon filename=$hvmsh_tmp_file noconf nodetail exio 2>>$error_log`;

  if ( $perfmon =~ /Failed/ ) {
    error( "HvmSh error: $perfmon " . __FILE__ . ":" . __LINE__ );
    exit(1);
  }

  my @input = split /(?=\[.*\])/, $perfmon;
  my $re = qr/^\[((MONITORING_INFORMATION)|(SYSTEM_USAGE_SUMMARY)|(SYSTEM_CPU_USAGE)
        |(LPAR_CPU_USAGE)|(SYSTEM_MEM_USAGE)|(PHYSICAL_NIC_USAGE)|(PHYSICAL_HBA_USAGE))\]\n.*\n/x;
  my $HVM_ID;
  my $product;

  for my $entry (@input) {
    my $header;
    $entry =~ s/$re//;
    my @table = map { my $s = $_; $s =~ s/^\s+//; $s } split /\n/, $entry;

    #MONITORING_INFORMATION
    if ($2) {
      ( $HVM_ID, $product ) = split /\t+/, $table[0];
      next;
    }

    for my $row (@table) {
      my @values = split /\s+/, $row;

      #SYSTEM_USAGE_SUMMARY
      if ($3) {
        my ( undef, $CAPACITY, $USED, undef, $INSUFF, $USED_p, undef, $INSUFF_p ) = @values;

        if ( $row =~ /^CPU/ ) {
          $message .= ":HSYS:CPU::$CAPACITY:$USED:$INSUFF:$USED_p:$INSUFF_p\:";
        }
        if ( $row =~ /^MEM/ ) {
          $message .= ":HSYS:MEM::$CAPACITY:$USED\::$USED_p\::";
        }
      }

      #SYSTEM_CPU_USAGE
      elsif ($4) {
        my ( undef, $COREs, undef, $CAPACITY, $USED, $USED_p, $USED_COREs ) = @values;

        if ( $row =~ /^(SYS1|SYS2)/ ) {
          $message .= ":HCPU:$1::$COREs\::$USED:$USED_p:$USED_COREs:";
        }
        if ( $row =~ /^(SHR_LPAR|DED_LPAR)/ ) {
          $message .= ":HCPU:$1::$COREs\:$CAPACITY:$USED:$USED_p:$USED_COREs:";
        }
      }

      #LPAR_CPU_USAGE
      elsif ($5) {
        my (
          undef, $NAME, undef,  undef, undef, $COREs, undef, undef, undef,
          $USED, undef, $DELAY, $IDLE, $IOW,  $NIOW,  undef, undef, undef
        ) = @values;
        my $uuid = defined $lpar_uuids{$NAME} ? $lpar_uuids{$NAME} : "";
        $message .= ":HLPAR:$NAME:$uuid:$COREs:$USED:$DELAY:$IDLE:$IOW:$NIOW";
      }

      # SYSTEM_MEM_USAGE
      elsif ($6) {
        my ( $NAME, $USED, $USED_p, $LPAR_USED_p ) = @values;
        $message .=
          $NAME =~ /SYS/
          ? ":HMEM:SYS::$USED:$USED_p\::::"
          : ":HMEM:LPAR::$USED:$USED_p:$LPAR_USED_p\:::";
      }

      # PHYSICAL_NIC_USAGE
      elsif ($7) {
        my ( $SID, $P, undef, undef, undef, undef, undef, undef, undef, undef, $R_BYTE, $S_BYTE, $T_BYTE, $R_PACKET, $S_PACKET, $T_PACKET, undef ) = @values;
        $message .= ":HNIC:$SID-$P\::$R_BYTE:$S_BYTE:$T_BYTE:$R_PACKET:$S_PACKET:$T_PACKET";
      }

      # PHYSICAL_HBA_USAGE
      elsif ($8) {
        my ( $SID, $P, undef, undef, undef, undef, undef, undef, undef, undef, $R_BYTE, $W_BYTE, $T_BYTE, $R_FRAME, $W_FRAME, $T_FRAME, undef ) = @values;

        unless ( $R_BYTE eq '*' || $W_BYTE eq '*' || $T_BYTE eq '*' || $R_FRAME eq '*' || $W_FRAME eq '*' || $T_FRAME eq '*' ) {
          $message .= ":HHBA:$SID-$P\::$R_BYTE:$W_BYTE:$T_BYTE:$R_FRAME:$W_FRAME:$T_FRAME";
        }
      }
    }
  }

  $message = "HITACHI:$HVM_ID:$product:$act_time_u:$act_time version $version\::::" . $message;
  return $message;
}

sub getHmcListFromJson {
  my @hmc_list = ();
  if ( !defined $ENV{INPUTDIR} ) {
    return @hmc_list;
  }
  eval {
    require JSON;
    1;
  } or do {
    return @hmc_list;
  };
  my $file = "$ENV{INPUTDIR}/etc/web_config/hosts.json";
  my $json = JSON->new->utf8;
  my $string;
  my $conf;

  {
    local $/ = undef;
    open( FILE, '<', $file ) or error( "Couldn't open file $file $!" . __FILE__ . ":" . __LINE__ ) && return @hmc_list;
    $string = <FILE>;
    close FILE;
  }

  if ($string) {
    eval {
      $conf = $json->decode($string);
      1;
    } or do {
      return @hmc_list;
    };
  }

  if ( !defined $conf || $conf eq "" ) {
    warn("Couldn't load hosts.json configuration. Returning empty array as hmc_list") && return @hmc_list;
  }

  foreach my $hmc_alias ( keys %{ $conf->{'platforms'}{'IBM Power Systems'}{'aliases'} } ) {
    my $hmc      = $conf->{'platforms'}{'IBM Power Systems'}{'aliases'}{$hmc_alias};
    my $hmc_host = $hmc->{host};
    foreach my $conf_key ( keys %{$hmc} ) {
      my $conf_value = $hmc->{$conf_key};
      $hmc_conf{$hmc_host}{$conf_key} = $conf_value;
    }
    push @hmc_list, $hmc_host;
  }
  return @hmc_list;
}

sub cpu_queue {

  my $vmstat  = shift;
  my $os      = shift;
  my $wpar_id = shift;
  my @vmstat  = @{$vmstat};

  if ( $os =~ m/Linux/ ) {
    my $uptime_out = `uptime`;
    $uptime_out =~ s/^\s+|\s+$//g;
    my @uptime = split /load average:/, $uptime_out;
    $uptime[1] =~ s/^\s+|\s+$//g;
    my @load = split / /, $uptime[1];

    my $load_parsed = substr $load[0], 0, -2;
    $load_parsed =~ s/,/./g;
    my $cpu_core = `grep "^processor" /proc/cpuinfo| wc -l`;
    $cpu_core =~ s/^\s+|\s+$//g;

    $vmstat[1] =~ s/^\s+//g;
    my @blocked     = split( / +/, $vmstat[1] );
    my $blocked_cnt = @blocked;

    my $blocked_indx   = "";
    my $raw_indx       = "";
    my $direct_io_indx = "";
    for ( my $i = 0; $i < $blocked_cnt; $i++ ) {
      if ( $blocked[$i] eq "b" ) {
        $blocked_indx = $i;
      }
    }
    $vmstat[3] =~ s/^\s+//g;
    my @data = split( / +/, $vmstat[3] );
    return ":queue_cpu:::$load_parsed:$cpu_core:$data[$blocked_indx]:::";
  }
  elsif ( $os =~ m/SunOS/ ) {

    my $uptime_out = `uptime`;
    $uptime_out =~ s/^\s+|\s+$//g;
    my @uptime = split /load average:/, $uptime_out;
    $uptime[1] =~ s/^\s+//g;
    my @load = split / /, $uptime[1];

    my $load_parsed = substr $load[0], 0, -2;
    $load_parsed =~ s/,/./g;
    my @cpu_core = `psrinfo -vp`;

    my $cpu_cnt  = @cpu_core;
    my $proc_num = 0;
    for ( my $i = 0; $i < $cpu_cnt; $i++ ) {
      if ( $cpu_core[$i] =~ m/processor/ ) {
        if ( $cpu_core[$i] =~ m/The core has / ) {next}
        my ($num) = $cpu_core[$i] =~ m/\s+\d+\s+virtual processor/g;
        $num =~ s/[a-z]|\s+//g;
        $proc_num += $num;
      }
    }

    $vmstat[2] =~ s/^\s+//g;
    my @blocked = split /\s+/, $vmstat[2];
    return ":queue_cpu:::$load_parsed:$proc_num:$blocked[2]:::";
  }
  elsif ( !($wpar_id eq '') && isdigit($wpar_id) && $wpar_id > 0 ) {
    my $uptime_out = `uptime`;
    $uptime_out =~ s/^\s+|\s+$//g;
    my @uptime = split /load average:/, $uptime_out;
    $uptime[1] =~ s/^\s+|\s+$//g;
    my @load = split / /, $uptime[1];

    my $load_parsed = substr $load[0], 0, -2;
    $load_parsed =~ s/,/./g;
    my $vmstat_cnt = @vmstat;

    my @cpu     = split /lcpu=/, $vmstat[1];
    my @cpu_cnt = split / /,     $cpu[1];

    $vmstat[6] =~ s/^\s+//g;
    my @blocked     = split( / +/, $vmstat[6] );
    my $blocked_cnt = @blocked;

    my $blocked_indx   = "";
    my $raw_indx       = "";
    my $direct_io_indx = "";
    for ( my $i = 0; $i < $blocked_cnt; $i++ ) {
      if ( $blocked[$i] eq "b" ) {
        $blocked_indx = $i;
      }
      elsif ( $blocked[$i] eq "p" ) {
        $raw_indx = $i;
      }
      elsif ( $blocked[$i] eq "w" ) {
        $direct_io_indx = $i;
      }
    }
    $vmstat[7] =~ s/^\s+//g;
    my @data = split( / +/, $vmstat[7] );
    return ":queue_cpu_aix:::$load_parsed:$cpu_cnt[0]:$data[$blocked_indx]:$data[$raw_indx]:$data[$direct_io_indx]:";
  }
  else {
    my $uptime_out = `uptime`;
    $uptime_out =~ s/^\s+|\s+$//g;
    my @uptime = split /load average:/, $uptime_out;
    $uptime[1] =~ s/^\s+|\s+$//g;
    my @load = split / /, $uptime[1];

    my $load_parsed = substr $load[0], 0, -2;
    $load_parsed =~ s/,/./g;
    my $vmstat_cnt = @vmstat;

    my @cpu     = split /lcpu=/, $vmstat[1];
    my @cpu_cnt = split / /,     $cpu[1];

    $vmstat[5] =~ s/^\s+//g;
    my @blocked     = split( / +/, $vmstat[5] );
    my $blocked_cnt = @blocked;

    my $blocked_indx   = "";
    my $raw_indx       = "";
    my $direct_io_indx = "";
    for ( my $i = 0; $i < $blocked_cnt; $i++ ) {
      if ( $blocked[$i] eq "b" ) {
        $blocked_indx = $i;
      }
      elsif ( $blocked[$i] eq "p" ) {
        $raw_indx = $i;
      }
      elsif ( $blocked[$i] eq "w" ) {
        $direct_io_indx = $i;
      }
    }
    $vmstat[6] =~ s/^\s+//g;
    my @data = split( / +/, $vmstat[6] );
    return ":queue_cpu_aix:::$load_parsed:$cpu_cnt[0]:$data[$blocked_indx]:$data[$raw_indx]:$data[$direct_io_indx]:";
  }
}

sub multipath_aix {

  my $LSPATH        = shift;
  my $os            = shift;
  my @exclude_disks = "";
  if ( -f "$multipath_exclude_file" ) { # list of excludes disk
    open( FH, "< $multipath_exclude_file" ) || error( " Can't open $multipath_exclude_file: $! " . __FILE__ . ":" . __LINE__ ) && next;
    my @data_from_file = <FH>;
    close(FH);
    @exclude_disks = grep {!/^\#/} @data_from_file;
  }
  my %hash_multi = ();
  my @ls_com     = `$LSPATH`;
  open( LS, "> $lsstat_data_file" ) || error( "Cannot open $lsstat_data_file: $!" . __FILE__ . ":" . __LINE__ ) && exit 1;
  foreach my $ls_line (@ls_com) {
    chomp $ls_line;
    my ( $name, $path_id, $connection, $parent, $path_status, $status ) = split( /\|/, $ls_line );
    my $grep_disk = grep /^$name$/, @exclude_disks;
    if ($grep_disk == 1) {
      print "multipath-exclude.txt file exists: skipping disk $name\n";
      next;
    } # if want to skip some disks
    $name         =~ s/:/=====double-colon=====/g;
    $path_id      =~ s/:/=====double-colon=====/g;
    $connection   =~ s/:/=====double-colon=====/g;
    $path_status  =~ s/:/=====double-colon=====/g;
    $status       =~ s/:/=====double-colon=====/g;
    if (defined $path_id && $path_id ne "") {
      if (!exists $hash_multi{$name}{'path_id'}) {
        $hash_multi{$name}{'path_id'} = [];
      }
      push @{$hash_multi{$name}{'path_id'}}, $path_id;
    }
    if (defined $connection && $connection ne "") {
      if (!exists $hash_multi{$name}{'connection'}) {
        $hash_multi{$name}{'connection'} = [];
      }
      push @{$hash_multi{$name}{'connection'}}, $connection;
    }
    if (defined $parent && $parent ne "") {
      if (!exists $hash_multi{$name}{'parent'}) {
        $hash_multi{$name}{'parent'} = [];
      }
      push @{$hash_multi{$name}{'parent'}}, $parent;
    }
    if (defined $path_status && $path_status ne "") {
      if (!exists $hash_multi{$name}{'path_status'}) {
        $hash_multi{$name}{'path_status'} = [];
      }
      push @{$hash_multi{$name}{'path_status'}}, $path_status;
    }
    if (defined $status && $status ne "") {
      if (!exists $hash_multi{$name}{'status'}) {
        $hash_multi{$name}{'status'} = [];
      }
      push @{$hash_multi{$name}{'status'}}, $status;
    }
  }

  foreach my $hdisk (keys %hash_multi) {
    my $inner_hash = $hash_multi{$hdisk};
    my @path_status_values = @{$inner_hash->{'path_status'}};
    my @status_values      = @{$inner_hash->{'status'}};

    my $count_ok1     = grep { $_ =~ /Available|Enabled/ } @path_status_values;
    my $count_nok1    = grep { $_ =~ /Failed|Disabled|N\/A|Missing/ } @path_status_values;
    my $count_ok2     = grep { $_ =~ /Available|Enabled/ } @status_values;
    my $count_nok2    = grep { $_ =~ /Failed|Disabled|N\/A|Missing/ } @status_values;
    my $final_status = "";

    # OK status
    if ( $count_ok1 >= 1 && $count_ok2 >= 1 && $count_nok1 == 0 && $count_nok2 == 0 ) {
      $final_status = "OK";
    }
    # Warning status
    if ($count_ok1 > $count_nok1 && $count_ok2 == $count_nok2){
      $final_status = "Warning";
    }
    # Critical status
    elsif ($count_ok1 == 0 && $count_ok2 == 0){
      $final_status = "Critical";
    }
    elsif ($count_ok1 == $count_nok1 && $count_ok2 == $count_nok2 ){
      $final_status = "Critical";
    }
    elsif ( $count_ok1 >= 1 && $count_nok1 == 0 && $count_nok1 == 0 && $count_nok2 >= 1 ) {
      $final_status = "Critical";
    }

    my $path_ids      = $inner_hash->{'path_id'};
    my $parents       = $inner_hash->{'parent'};
    my $connections   = $inner_hash->{'connection'};
    my $statuses      = $inner_hash->{'status'};
    my $path_statuses = $inner_hash->{'path_status'};

    for my $i (0 .. $#{$path_ids}) {
      my $path_id     = $path_ids->[$i];
      my $parent      = $parents->[$i];
      my $connection  = $connections->[$i];
      my $status      = $statuses->[$i];
      my $path_status = $path_statuses->[$i];
      print LS"$hdisk:$path_id:$parent:$connection:$path_status:$status:$final_status\n";
    }
  }
  close(LS);
  return 1;

}

sub multipath_linux {
  my $MULTI_LIN = shift;
  my $os        = shift;
  my @MULTI     = `$MULTI_LIN 2>/dev/null`;
  my ( $alias, $product, $size, $policy_status, $paths );
  my $i   = 0;
  my $sep = "/";
  my $string1;
  my $string2;
  my $string3;
  open( MULTI_L, "> $multi_lin_data_file" ) || error( "Cannot open $multi_lin_data_file: $!" . __FILE__ . ":" . __LINE__ ) && exit 1;

  foreach my $line (@MULTI) {
    chomp $line;
    $line =~ s/^\s+//g;

    #my ($alias,$product,$size,$policy_status);
    if ( $line =~ /.*,.*$/ ) {
      ( $alias, $product ) = split( /,/, $line );
      #$alias =~ s/\(/\\(/g;
      #$alias =~ s/\)/\\)/g;
      $alias =~ s/:/=====double-colon=====/g;
    }
    if ( $line =~ /^size/ ) {
      $size = $line;
      $size =~ s/:/=====double-colon=====/g;
    }
    if ( $line =~ /policy/ ) {
      $policy_status = $line;
      $policy_status =~ s/^\s+//g;
      $policy_status =~ s/\|\-\+\-//g;
      $policy_status =~ s/\|//g;
      $policy_status =~ s/\`\-\+\-//g;
      $policy_status =~ s/^\s+//g;
      $string1 .= "$policy_status" . "$sep";
      $string1 =~ s/:/=====double-colon=====/g;
    }
    if ( $line =~ /\d\:\d\:\d\:\d/ ) {
      $paths = $line;
      $paths =~ s/^\s+//g;
      $paths =~ s/\`\-//g;
      $paths =~ s/\|//g;
      $paths =~ s/|-+-//g;
      $paths =~ s/^\s+//g;

      #$path_line    =~ s/(active \S+ \S+)|(failed \S+ \S+)|(faulty \S+ \S+)//g;
      $string2 .= "$paths" . "$sep";
      $string2 =~ s/:/=====double-colon=====/g;
      if ( $line =~ /^`-/ ) {
        print MULTI_L"$alias:$size:$string1:$string2\n";
        $string1 = "";
        $string2 = "";
      }
    }
    $i++;
  }
  close(MULTI_L);
  return 1;
}

sub multipath_solaris {

  my $MULTI_SOL = shift;
  my $os        = shift;
  my @MULTI     = `$MULTI_SOL`;
  my @all_path  = grep {/\/dev\//} @MULTI;
  my @exclude_disks = "";
  if ( -f "$multipath_exclude_file" ) { # list of excludes disk
    open( FH, "< $multipath_exclude_file" ) || error( " Can't open $multipath_exclude_file: $! " . __FILE__ . ":" . __LINE__ ) && next;
    my @data_from_file = <FH>;
    close(FH);
    @exclude_disks = grep {!/^\#/} @data_from_file;
  }
  open( MULTI_S, "> $multi_sol_data_file" ) || error( "Cannot open $multi_sol_data_file: $!" . __FILE__ . ":" . __LINE__ ) && exit 1;
  foreach my $path (@all_path) {
    chomp $path;
    $path =~ s/^\s+//g;
    my @multi_single = `mpathadm show lu $path`;
    my $disk_alias   = $path;
    my $grep_disk = grep /^$disk_alias$/, @exclude_disks;
    if ($grep_disk == 1) {
      print "multipath-exclude.txt file exists: skipping disk $disk_alias\n";
      next;
    } # if want to skip some disks
    $disk_alias =~ s/\/dev\/rdsk\///g;
    my ($disk_id)           = grep {/Name:/} @multi_single;
    my ($vendor)            = grep {/Vendor:/} @multi_single;
    my ($product)           = grep {/Product:/} @multi_single;
    my ($revision)          = grep {/Revision:/} @multi_single;
    my ($name_type)         = grep {/Name Type:/} @multi_single;
    my ($asymmetric)        = grep {/Asymmetric:/} @multi_single;
    my ($curr_load_balance) = grep {/Current Load Balance:/} @multi_single;

    $disk_id           =~ s/\s+//g;
    $disk_id           =~ s/Name://g;
    $vendor            =~ s/\s+//g;
    $vendor            =~ s/Vendor://g;
    $product           =~ s/\s+//g;
    $product           =~ s/Product://g;
    $revision          =~ s/\s+//g;
    $revision          =~ s/Revision://g;
    $name_type         =~ s/\s+//g;
    $name_type         =~ s/NameType://g;
    $asymmetric        =~ s/\s+//g;
    $asymmetric        =~ s/Asymmetric://g;
    $curr_load_balance =~ s/\s+//g;
    $curr_load_balance =~ s/CurrentLoadBalance://g;

    my (@rel_ids)       = grep {/Relative ID:/} @multi_single;
    my (@disableds)     = grep {/Disabled:/} @multi_single;
    my (@path_states)   = grep {/Path State:/} @multi_single;
    my (@access_states) = grep {/Access State:/} @multi_single;

    my $sep     = "/";
    my $string1 = "";
    foreach my $rel_id (@rel_ids) {
      $rel_id =~ s/\s+//g;
      $rel_id =~ s/RelativeID://g;
      $string1 .= "$rel_id" . "$sep";
    }
    my $string2 = "";
    foreach my $disabled (@disableds) {
      $disabled =~ s/\s+//g;
      $disabled =~ s/Disabled://g;
      $string2 .= "$disabled" . "$sep";
    }
    my $string3 = "";
    foreach my $path_state (@path_states) {
      $path_state =~ s/\s+//g;
      $path_state =~ s/PathState://g;
      $string3 .= "$path_state" . "$sep";
    }
    my $string4 = "";
    foreach my $access_state (@access_states) {
      $access_state =~ s/\s+//g;
      $access_state =~ s/AccessState://g;
      $string4 .= "$access_state" . "$sep";
    }

    #print "disk_id-$disk_id,vendor-$vendor,product-$product,revision-$revision,name_type-$name_type,asymetric-$asymmetric,curr-$curr_load_balance,string1-$string1,string2-$string2,string3-$string3,string4-$string4\n";
    print MULTI_S"$disk_id:$disk_alias:$vendor:$product:$revision:$name_type:$asymmetric:$curr_load_balance:$string1:$string2:$string3:$string4\n";

  }
  close(MULTI_S);
  return 1;

}

# simple replacement of copy subroutine from File::Copy ( module missing in RHEL9+ base-perl package :-/ )
sub copy {
    if (@_ != 2) {
        warn( "Usage: copy(FROM, TO) " );
        return 0;
    }

    my $from = shift;
    my $to = shift;

    if ( !open( IN,  "< $from" ) ) {
        warn( "can't open $from $!" );
        return 0;
    }
    if ( !open( OUT, "> $to" ) ) {
        warn( "can't open $to $!" );
        return 0;
    }

    my $blksize = ( stat IN )[11] || 16384;          # preferred block size?
    while ( my $len = sysread IN, my $buf, $blksize ) {
        if ( !defined $len ) {
            next if $! =~ /^Interrupted/;       # ^Z and fg
            warn( "System read error: $!" );
            return 0;
        }
        my $offset = 0;
        while ($len) {          # Handle partial writes.
            my $written;
            if ( !defined( $written = syswrite OUT, $buf, $len, $offset ) ) {
                warn( "System write error: $!" );
                return 0;
            }
            $len    -= $written;
            $offset += $written;
        };
    }

    close(IN);
    close(OUT);
}

sub get_uuid {
  my $disk_name = shift;
  my $uuid = "";
  my $name_server = "";

  my @volumegrep  = `lscfg -vl $disk_name`;
  ($uuid) = grep ( /Serial Number/, @volumegrep );
  if ( !defined $uuid || $uuid eq '' ) {
    ($uuid) = grep ( /\(Z1\)/, @volumegrep );
  }
  ($name_server) = grep ( /\(Z7\)/, @volumegrep );
  if ( !defined $uuid ) {
    print "lsvcfg -vl $disk_name : skipping $disk_name as disk UUID is not defined, will trying odmget \n";
  }
  else {
    $uuid        =~ s/Device Specific\.\(Z1\)\.\.\.\.\.\.\.\.//g;
    $uuid        =~ s/Serial Number\.\.\.\.\.\.\.\.\.\.\.\.\.\.\.//g;
    $uuid        =~ s/^\s+//g;
    chomp $uuid;
    if (defined $name_server && $name_server ne ""){
      $name_server =~ s/Device Specific\.\(Z7\)\.\.\.\.\.\.\.\.SysName\://g;
      $name_server =~ s/^\s+//g;
      chomp $name_server;
    }
  }
  if (!defined $uuid || length($uuid) < 20 ) {
    # it is most probably not volume ID, as they have 30+ characters usually like 600507640081801EC00000000005B8E2
    # use odmget instead
    print "odmget -q \"name=$disk_name and attribute=unique_id\" CuAt 2>/dev/null\n";
    my @SN=`odmget -q "name=$disk_name and attribute=unique_id" CuAt 2>/dev/null`;
    foreach my $line (@SN) {
      chomp($line);
      if ( $line =~ /^\s+value\s+=\s+"(\w+AIXvscsi)"/ ) {
        # this might not have to work in all cases
        $uuid = substr($1,9,32);
        print $disk_name.":".substr($1,9,32)."\n";
      }
      else {
        if ($line =~ /^\s+value\s+=\s+"(.*HITACHI.*)"/i){
          # for HITACHI
          # volume_wwn= 60060e8008ee69000050ee6900002b38
          my @wwn_hitachi = `odmget -q "name=$disk_name and attribute=ww_name" CuAt 2>/dev/null`;
          my $hitachi_id = "";
          foreach my $wwline (@wwn_hitachi) {
            chomp($wwline);
            if ($wwline =~ /^\s+value\s+=\s+"(0x[0-9a-fA-F]+)"/) {
              my $full_wwn = $1;
              my $without_prefix = substr($full_wwn, 2);
              my $trimmed_wwn = substr($without_prefix, 3);
              while (length($trimmed_wwn) < 15) {
                $trimmed_wwn .= "0";
              }
              $hitachi_id = $trimmed_wwn; # $hitachi_id=60e8008ee690000 
            }
          }
          my $full_id   = $1;
          my $serial_id = substr($full_id, 6, 6);  # $serial_id=50EE69
          my $ldev_id   = substr($full_id, 12, 4); # $ldev_id=2B38
          $ldev_id      = sprintf("%08s", $ldev_id);
          $uuid         = "600" . "$hitachi_id" . "$serial_id" . "$ldev_id";
          $uuid         = lc $uuid;
        }
        elsif ( $line =~ /^\s*value\s*=\s*"([\w\-]+fcp)"\s*$/ ) {
          # for IBM FC connected storage like DS8k, SVC, Storwize, FlashSystem
          $uuid = substr($1,5,32);
          print $disk_name.":".substr($1,5,32)."\n";
        }
        else {
          if ( $line =~ /^\s+value\s+=\s+"(.*)"/ ) {
            # value = "3211000377ce-0000000e0ECompellent Vol08COMPELNTfcp"
            # it is not always volume ID but we cannot do more, it is al least something
            $uuid = $line;
            $uuid =~ s/^.*value = "//g;
            $uuid =~ s/".*$//g;
          }
        }
      }
      # 3par:       value = "251000240000AC03000002VV083PARdatafcp" --> there is only part of volume id 60002AC000000000000000240000AC03, no idea how to get it
      # compellent: value = "3211000377ce-0000000d0ECompellent Vol08COMPELNTfcp" --> same here 6000d3100377ce00000000000000000d
      # huawei:     value = "362136e8abf3100afc23f9f7f1c320000081204xsg106huaweifcp" --> 6e8abf3100afc23f9f7f1c3200000812
    }
  }

  if ( !defined $uuid || $uuid eq "" || length($uuid) < 20 ) {
    print "$disk_name : volume ID has not been found, it must have more than 20 chars : $uuid\n";
    return "";
  }
  return wantarray ? ($uuid, $name_server) : $uuid;
}

sub basename {
  return ( split "\/", $_[0] )[-1];
}

sub parse_vg_lv {
  my ($full) = @_;
  if ($full =~ /^(.*?)(?<!-)-(?!-)(.*)$/) {
    my $vg = $1;
    my $lv = $2;
    $vg =~ s/--/-/g;
    $lv =~ s/--/-/g;
    return ($vg, $lv);
  }
  return ("?", "?");
}

sub get_wwn {
  my ($dev) = @_;
  $dev =~ s{^/dev/}{};
  return "unknown" unless $dev =~ /^sd/;

  my $path = "/sys/block/$dev/device/wwid";
  if (-e $path) {
    chomp(my $wwn = `cat $path 2>/dev/null`);
    if ($wwn) {
      $wwn =~ s/^naa\.//;
      return $wwn;
    }
  }
  return "unknown";
}

sub resolve_parents {
  my ($dm) = @_;
  my $slave_path = "/sys/block/$dm/slaves";
  return (["/dev/$dm"], ["unknown"]) unless -d $slave_path;

  my @parents;
  my @wwns;

  opendir(my $dh, $slave_path) or return (["/dev/$dm"], ["unknown"]);
  while (my $d = readdir($dh)) {
    next if $d eq '.' or $d eq '..';

    if ($d =~ /^sd/) {
      my $wwn = get_wwn("/dev/$d");
      push @parents, "/dev/$d";
      push @wwns, $wwn;
    }
    elsif ($d =~ /^dm-/) {
      my $uuid = `cat /sys/block/$d/dm/uuid 2>/dev/null`;
      chomp $uuid;
      if ($uuid =~ /^mpath-/) {
        my $name = `cat /sys/block/$d/dm/name 2>/dev/null`;
        chomp $name;

        my $spath = "/sys/block/$d/slaves";
        if (-d $spath) {
          opendir(my $sdh, $spath);
          my $found = 0;
          while (my $sd = readdir($sdh)) {
            next if $sd eq '.' or $sd eq '..';
            if ($sd =~ /^sd/) {
              my $wwn = get_wwn("/dev/$sd");
              push @parents, "/dev/mapper/$name";
              push @wwns, $wwn;
              $found = 1;
            }
          }
          closedir($sdh);
          if (!$found) {
            push @parents, "/dev/mapper/$name";
            push @wwns, "unknown";
          }
        } else {
          push @parents, "/dev/mapper/$name";
          push @wwns, "unknown";
        }
      }
    }
  }
  closedir($dh);

  return (\@parents, \@wwns);
}

sub debug_time {
  my ($label, $t0) = @_;
  my $dt = time() - $t0;
  printf STDERR "[DEBUG] %-30s %4ds\n", $label, $dt;
  return time();
}
